diff --git a/Makefile b/Makefile index 7dbc01b1..9a38f499 100644 --- a/Makefile +++ b/Makefile @@ -166,7 +166,8 @@ dev-setup: {'op': 'replace', 'path': '/webhooks/5/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/pods\",'caBundle':\"$${CA_BUNDLE}\"}},\ {'op': 'replace', 'path': '/webhooks/6/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/persistentvolumeclaims\",'caBundle':\"$${CA_BUNDLE}\"}},\ {'op': 'replace', 'path': '/webhooks/7/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/services\",'caBundle':\"$${CA_BUNDLE}\"}},\ - {'op': 'replace', 'path': '/webhooks/8/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/tenants\",'caBundle':\"$${CA_BUNDLE}\"}}\ + {'op': 'replace', 'path': '/webhooks/8/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/tenants\",'caBundle':\"$${CA_BUNDLE}\"}},\ + {'op': 'replace', 'path': '/webhooks/9/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/tenants\",'caBundle':\"$${CA_BUNDLE}\"}}\ ]" && \ kubectl patch crd tenants.capsule.clastix.io \ --type='json' -p="[\ diff --git a/api/v1beta2/tenant_types.go b/api/v1beta2/tenant_types.go index 93e01b75..3b43ce78 100644 --- a/api/v1beta2/tenant_types.go +++ b/api/v1beta2/tenant_types.go @@ -17,6 +17,8 @@ type TenantSpec struct { NamespaceOptions *NamespaceOptions `json:"namespaceOptions,omitempty"` // Specifies options for the Service, such as additional metadata or block of certain type of Services. Optional. ServiceOptions *api.ServiceOptions `json:"serviceOptions,omitempty"` + // Specifies options for the Pods deployed in the Tenant namespaces, such as additional metadata. + PodOptions *api.PodOptions `json:"podOptions,omitempty"` // Specifies the allowed StorageClasses assigned to the Tenant. // Capsule assures that all PersistentVolumeClaim resources created in the Tenant can use only one of the allowed StorageClasses. // A default value can be specified, and all the PersistentVolumeClaim resources created will inherit the declared class. diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 7245688f..119085f4 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -716,6 +716,11 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) { *out = new(api.ServiceOptions) (*in).DeepCopyInto(*out) } + if in.PodOptions != nil { + in, out := &in.PodOptions, &out.PodOptions + *out = new(api.PodOptions) + (*in).DeepCopyInto(*out) + } if in.StorageClasses != nil { in, out := &in.StorageClasses, &out.StorageClasses *out = new(api.DefaultAllowedListSpec) diff --git a/charts/capsule/crds/tenant-crd.yaml b/charts/capsule/crds/tenant-crd.yaml index 314016fb..8ae0d6ae 100644 --- a/charts/capsule/crds/tenant-crd.yaml +++ b/charts/capsule/crds/tenant-crd.yaml @@ -632,6 +632,22 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pod, such as additional metadata. Optional. + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule operator places on any Service resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object preventDeletion: description: Prevent accidental deletion of the Tenant. When enabled, the deletion request will be declined. type: boolean @@ -1737,6 +1753,22 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pod, such as additional metadata. Optional. + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule operator places on any Service resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object priorityClasses: description: Specifies the allowed priorityClasses assigned to the Tenant. Capsule assures that all Pods resources created in the Tenant @@ -2869,6 +2901,22 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pod, such as additional metadata. Optional. + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule operator places on any Service resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object preventDeletion: description: Prevent accidental deletion of the Tenant. When enabled, the deletion request will be declined. diff --git a/config/crd/bases/capsule.clastix.io_tenants.yaml b/config/crd/bases/capsule.clastix.io_tenants.yaml index 75c54723..a1d0f616 100644 --- a/config/crd/bases/capsule.clastix.io_tenants.yaml +++ b/config/crd/bases/capsule.clastix.io_tenants.yaml @@ -2859,6 +2859,24 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pods deployed in the Tenant + namespaces, such as additional metadata. + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule + operator places on any Pod resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object preventDeletion: description: Prevent accidental deletion of the Tenant. When enabled, the deletion request will be declined. diff --git a/config/install.yaml b/config/install.yaml index d753fe84..ad90fb18 100644 --- a/config/install.yaml +++ b/config/install.yaml @@ -2437,6 +2437,22 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pods deployed in the Tenant namespaces, such as additional metadata. + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule operator places on any Pod resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object preventDeletion: description: Prevent accidental deletion of the Tenant. When enabled, the deletion request will be declined. type: boolean diff --git a/controllers/pod/errors.go b/controllers/pod/errors.go new file mode 100644 index 00000000..ceafff4a --- /dev/null +++ b/controllers/pod/errors.go @@ -0,0 +1,30 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package pod + +import "fmt" + +type NonTenantObjectError struct { + objectName string +} + +func NewNonTenantObject(objectName string) error { + return &NonTenantObjectError{objectName: objectName} +} + +func (n NonTenantObjectError) Error() string { + return fmt.Sprintf("Skipping labels sync for %s as it doesn't belong to tenant", n.objectName) +} + +type NoPodMetadataError struct { + objectName string +} + +func NewNoPodMetadata(objectName string) error { + return &NoPodMetadataError{objectName: objectName} +} + +func (n NoPodMetadataError) Error() string { + return fmt.Sprintf("Skipping labels sync for %s because no AdditionalLabels or AdditionalAnnotations presents in Tenant spec", n.objectName) +} diff --git a/controllers/pod/metadata.go b/controllers/pod/metadata.go new file mode 100644 index 00000000..9c9d772f --- /dev/null +++ b/controllers/pod/metadata.go @@ -0,0 +1,130 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package pod + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierr "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/utils" +) + +type MetadataReconciler struct { + Client client.Client +} + +func (m *MetadataReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + var pod corev1.Pod + + logger := log.FromContext(ctx) + + tenant, err := m.getTenant(ctx, request.NamespacedName, m.Client) + if err != nil { + noTenantObjError := &NonTenantObjectError{} + noPodMetaError := &NoPodMetadataError{} + + if errors.As(err, &noTenantObjError) || errors.As(err, &noPodMetaError) { + return reconcile.Result{}, nil + } + + logger.Error(err, fmt.Sprintf("Cannot get tenant corev1.Pod %s/%s", request.Namespace, request.Name)) + + return reconcile.Result{}, err + } + + err = m.Client.Get(ctx, request.NamespacedName, &pod) + if err != nil { + if apierr.IsNotFound(err) { + return reconcile.Result{}, nil + } + + return reconcile.Result{}, err + } + + _, err = controllerutil.CreateOrUpdate(ctx, m.Client, &pod, func() (err error) { + pod.SetLabels(m.sync(pod.GetLabels(), tenant.Spec.PodOptions.AdditionalMetadata.Labels)) + pod.SetAnnotations(m.sync(pod.GetAnnotations(), tenant.Spec.PodOptions.AdditionalMetadata.Annotations)) + + return nil + }) + + return reconcile.Result{}, err +} + +func (m *MetadataReconciler) getTenant(ctx context.Context, namespacedName types.NamespacedName, client client.Client) (*capsulev1beta2.Tenant, error) { + ns := &corev1.Namespace{} + tenant := &capsulev1beta2.Tenant{} + + if err := client.Get(ctx, types.NamespacedName{Name: namespacedName.Namespace}, ns); err != nil { + return nil, err + } + + capsuleLabel, _ := utils.GetTypeLabel(&capsulev1beta2.Tenant{}) + if _, ok := ns.GetLabels()[capsuleLabel]; !ok { + return nil, NewNonTenantObject(namespacedName.Name) + } + + if err := client.Get(ctx, types.NamespacedName{Name: ns.Labels[capsuleLabel]}, tenant); err != nil { + return nil, err + } + + if tenant.Spec.PodOptions == nil || tenant.Spec.PodOptions.AdditionalMetadata == nil { + return nil, NewNoPodMetadata(namespacedName.Name) + } + + return tenant, nil +} + +func (m *MetadataReconciler) sync(available map[string]string, tenantSpec map[string]string) map[string]string { + if tenantSpec != nil { + if available == nil { + return tenantSpec + } + + for key, value := range tenantSpec { + if available[key] != value { + available[key] = value + } + } + } + + return available +} + +func (m *MetadataReconciler) forOptionPerInstanceName(ctx context.Context) builder.ForOption { + return builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { + return m.isNamespaceInTenant(ctx, object.GetNamespace()) + })) +} + +func (m *MetadataReconciler) isNamespaceInTenant(ctx context.Context, namespace string) bool { + tl := &capsulev1beta2.TenantList{} + if err := m.Client.List(ctx, tl, client.MatchingFieldsSelector{ + Selector: fields.OneTermEqualSelector(".status.namespaces", namespace), + }); err != nil { + return false + } + + return len(tl.Items) > 0 +} + +func (m *MetadataReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Pod{}, m.forOptionPerInstanceName(ctx)). + Complete(m) +} diff --git a/docs/content/general/crds-apis.md b/docs/content/general/crds-apis.md index 696a41eb..29a5d794 100644 --- a/docs/content/general/crds-apis.md +++ b/docs/content/general/crds-apis.md @@ -2963,6 +2963,13 @@ TenantSpec defines the desired state of Tenant. Specifies the label to control the placement of pods on a given pool of worker nodes. All namespaces created within the Tenant will have the node selector annotation. This annotation tells the Kubernetes scheduler to place pods on the nodes having the selector label. Optional.
false + + podOptions + object + + Specifies options for the Pods deployed in the Tenant namespaces, such as additional metadata.
+ + false preventDeletion boolean @@ -4397,6 +4404,65 @@ NetworkPolicyPort describes a port to allow traffic on +### Tenant.spec.podOptions + + + +Specifies options for the Pods deployed in the Tenant namespaces, such as additional metadata. + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
additionalMetadataobject + Specifies additional labels and annotations the Capsule operator places on any Pod resource in the Tenant. Optional.
+ 		false
+ + +### Tenant.spec.podOptions.additionalMetadata + + + +Specifies additional labels and annotations the Capsule operator places on any Pod resource in the Tenant. Optional. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
annotationsmap[string]string +
+ 		false
labelsmap[string]string +
+ 		false
+ + ### Tenant.spec.priorityClasses diff --git a/e2e/pod_metadata_test.go b/e2e/pod_metadata_test.go new file mode 100644 index 00000000..98722cb7 --- /dev/null +++ b/e2e/pod_metadata_test.go @@ -0,0 +1,115 @@ +//go:build e2e + +// Copyright 2020-2021 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("adding metadata to Pod objects", func() { + tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-metadata", + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: capsulev1beta2.OwnerListSpec{ + { + Name: "gatsby", + Kind: "User", + }, + }, + PodOptions: &api.PodOptions{ + AdditionalMetadata: &api.AdditionalMetadataSpec{ + Labels: map[string]string{ + "k8s.io/custom-label": "foo", + "clastix.io/custom-label": "bar", + }, + Annotations: map[string]string{ + "k8s.io/custom-annotation": "bizz", + "clastix.io/custom-annotation": "buzz", + }, + }, + }, + }, + } + + JustBeforeEach(func() { + EventuallyCreation(func() error { + + tnt.ResourceVersion = "" + return k8sClient.Create(context.TODO(), tnt) + }).Should(Succeed()) + }) + + JustAfterEach(func() { + Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + }) + + It("should apply them to Pod", func() { + ns := NewNamespace("") + NamespaceCreation(ns, tnt.Spec.Owners[0], defaultTimeoutInterval).Should(Succeed()) + fmt.Sprint("namespace created") + //TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + fmt.Sprint("tenant contains list namespace") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-metadata", + Namespace: ns.GetName(), + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container", + Image: "quay.io/google-containers/pause-amd64:3.0", + ImagePullPolicy: "IfNotPresent", + }, + }, + RestartPolicy: "Always", + }, + } + + EventuallyCreation(func() (err error) { + _, err = ownerClient(tnt.Spec.Owners[0]).CoreV1().Pods(ns.GetName()).Create(context.Background(), pod, metav1.CreateOptions{}) + + return + }).Should(Succeed()) + + By("checking additional labels", func() { + Eventually(func() (ok bool) { + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: pod.GetName(), Namespace: ns.GetName()}, pod)).Should(Succeed()) + for k, v := range tnt.Spec.PodOptions.AdditionalMetadata.Labels { + ok, _ = HaveKeyWithValue(k, v).Match(pod.GetLabels()) + if !ok { + return false + } + } + return true + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + }) + + By("checking additional annotations", func() { + Eventually(func() (ok bool) { + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: pod.GetName(), Namespace: ns.GetName()}, pod)).Should(Succeed()) + for k, v := range tnt.Spec.PodOptions.AdditionalMetadata.Annotations { + ok, _ = HaveKeyWithValue(k, v).Match(pod.GetAnnotations()) + if !ok { + return false + } + } + return true + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + }) + }) + +}) diff --git a/main.go b/main.go index d0f7190d..4ae07cea 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,7 @@ import ( capsulev1beta1 "github.com/projectcapsule/capsule/api/v1beta1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" configcontroller "github.com/projectcapsule/capsule/controllers/config" + podlabelscontroller "github.com/projectcapsule/capsule/controllers/pod" "github.com/projectcapsule/capsule/controllers/pv" rbaccontroller "github.com/projectcapsule/capsule/controllers/rbac" "github.com/projectcapsule/capsule/controllers/resources" @@ -291,6 +292,11 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels") } + if err = (&podlabelscontroller.MetadataReconciler{Client: manager.GetClient()}).SetupWithManager(ctx, manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "PodLabels") + os.Exit(1) + } + if err = (&pv.Controller{}).SetupWithManager(manager); err != nil { setupLog.Error(err, "unable to create controller", "controller", "PersistentVolume") os.Exit(1) diff --git a/pkg/api/pod_options.go b/pkg/api/pod_options.go new file mode 100644 index 00000000..4c122e02 --- /dev/null +++ b/pkg/api/pod_options.go @@ -0,0 +1,11 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package api + +// +kubebuilder:object:generate=true + +type PodOptions struct { + // Specifies additional labels and annotations the Capsule operator places on any Pod resource in the Tenant. Optional. + AdditionalMetadata *AdditionalMetadataSpec `json:"additionalMetadata,omitempty"` +} diff --git a/pkg/api/zz_generated.deepcopy.go b/pkg/api/zz_generated.deepcopy.go index 70c8265b..d09c76c3 100644 --- a/pkg/api/zz_generated.deepcopy.go +++ b/pkg/api/zz_generated.deepcopy.go @@ -213,6 +213,26 @@ func (in *NetworkPolicySpec) DeepCopy() *NetworkPolicySpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodOptions) DeepCopyInto(out *PodOptions) { + *out = *in + if in.AdditionalMetadata != nil { + in, out := &in.AdditionalMetadata, &out.AdditionalMetadata + *out = new(AdditionalMetadataSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodOptions. +func (in *PodOptions) DeepCopy() *PodOptions { + if in == nil { + return nil + } + out := new(PodOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceQuotaSpec) DeepCopyInto(out *ResourceQuotaSpec) { *out = *in