diff --git a/controllers/tenant/namespaces.go b/controllers/tenant/namespaces.go index 00b2f64a..73e2db38 100644 --- a/controllers/tenant/namespaces.go +++ b/controllers/tenant/namespaces.go @@ -17,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" capsulev1beta1 "github.com/clastix/capsule/api/v1beta1" + "github.com/clastix/capsule/pkg/utils" ) // Ensuring all annotations are applied to each Namespace handled by the Tenant. @@ -72,11 +73,7 @@ func (r *Manager) syncNamespaceMetadata(ctx context.Context, namespace string, t } if tnt.Spec.NodeSelector != nil { - var selector []string - for k, v := range tnt.Spec.NodeSelector { - selector = append(selector, fmt.Sprintf("%s=%s", k, v)) - } - annotations["scheduler.alpha.kubernetes.io/node-selector"] = strings.Join(selector, ",") + annotations = utils.BuildNodeSelector(tnt, annotations) } if tnt.Spec.IngressOptions.AllowedClasses != nil { diff --git a/main.go b/main.go index 7e89b52e..1bba13fe 100644 --- a/main.go +++ b/main.go @@ -205,7 +205,7 @@ func main() { route.Service(service.Handler()), route.NetworkPolicy(utils.InCapsuleGroups(cfg, networkpolicy.Handler())), route.Tenant(tenant.NameHandler(), tenant.RoleBindingRegexHandler(), tenant.IngressClassRegexHandler(), tenant.StorageClassRegexHandler(), tenant.ContainerRegistryRegexHandler(), tenant.HostnameRegexHandler(), tenant.FreezedEmitter(), tenant.ServiceAccountNameHandler(), tenant.ForbiddenAnnotationsRegexHandler(), tenant.ProtectedHandler()), - route.OwnerReference(utils.InCapsuleGroups(cfg, ownerreference.Handler(cfg))), + route.OwnerReference(utils.InCapsuleGroups(cfg, namespacewebhook.OwnerReferenceHandler(), ownerreference.Handler(cfg))), route.Cordoning(tenant.CordoningHandler(cfg), tenant.ResourceCounterHandler()), route.Node(utils.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))), ) diff --git a/pkg/utils/node_selector.go b/pkg/utils/node_selector.go new file mode 100644 index 00000000..d7160e87 --- /dev/null +++ b/pkg/utils/node_selector.go @@ -0,0 +1,35 @@ +// Copyright 2020-2021 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "fmt" + "sort" + "strings" + + capsulev1beta1 "github.com/clastix/capsule/api/v1beta1" +) + +const ( + NodeSelectorAnnotation = "scheduler.alpha.kubernetes.io/node-selector" +) + +func BuildNodeSelector(tnt *capsulev1beta1.Tenant, nsAnnotations map[string]string) map[string]string { + if nsAnnotations == nil { + nsAnnotations = make(map[string]string) + } + + selector := make([]string, 0, len(tnt.Spec.NodeSelector)) + + for k, v := range tnt.Spec.NodeSelector { + selector = append(selector, fmt.Sprintf("%s=%s", k, v)) + } + // Sorting the resulting slice: iterating over maps is randomized, and we could end-up + // in multiple reconciliations upon multiple node selectors. + sort.Strings(selector) + + nsAnnotations[NodeSelectorAnnotation] = strings.Join(selector, ",") + + return nsAnnotations +} diff --git a/pkg/webhook/namespace/freezed.go b/pkg/webhook/namespace/freezed.go index edda21e2..290217a4 100644 --- a/pkg/webhook/namespace/freezed.go +++ b/pkg/webhook/namespace/freezed.go @@ -69,7 +69,7 @@ func (r *freezedHandler) OnDelete(c client.Client, _ *admission.Decoder, recorde tnt := tntList.Items[0] - if tnt.IsCordoned() && utils.IsCapsuleUser(req, r.configuration.UserGroups()) { + if tnt.IsCordoned() && utils.IsCapsuleUser(ctx, req, c, r.configuration.UserGroups()) { recorder.Eventf(&tnt, corev1.EventTypeWarning, "TenantFreezed", "Namespace %s cannot be deleted, the current Tenant is freezed", req.Name) response := admission.Denied("the selected Tenant is freezed") @@ -101,7 +101,7 @@ func (r *freezedHandler) OnUpdate(c client.Client, decoder *admission.Decoder, r tnt := tntList.Items[0] - if tnt.IsCordoned() && utils.IsCapsuleUser(req, r.configuration.UserGroups()) { + if tnt.IsCordoned() && utils.IsCapsuleUser(ctx, req, c, r.configuration.UserGroups()) { recorder.Eventf(&tnt, corev1.EventTypeWarning, "TenantFreezed", "Namespace %s cannot be updated, the current Tenant is freezed", ns.GetName()) response := admission.Denied("the selected Tenant is freezed") diff --git a/pkg/webhook/namespace/owner_reference.go b/pkg/webhook/namespace/owner_reference.go new file mode 100644 index 00000000..b38dd40c --- /dev/null +++ b/pkg/webhook/namespace/owner_reference.go @@ -0,0 +1,64 @@ +// Copyright 2020-2021 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package namespace + +import ( + "context" + "fmt" + "net/http" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulewebhook "github.com/clastix/capsule/pkg/webhook" + "github.com/clastix/capsule/pkg/webhook/utils" +) + +type ownerReferenceHandler struct{} + +func OwnerReferenceHandler() capsulewebhook.Handler { + return &ownerReferenceHandler{} +} + +func (r *ownerReferenceHandler) OnCreate(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return nil + } +} + +func (r *ownerReferenceHandler) OnDelete(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return nil + } +} + +func (r *ownerReferenceHandler) OnUpdate(_ client.Client, decoder *admission.Decoder, _ record.EventRecorder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + oldNs := &corev1.Namespace{} + if err := decoder.DecodeRaw(req.OldObject, oldNs); err != nil { + return utils.ErroredResponse(err) + } + + newNs := &corev1.Namespace{} + if err := decoder.Decode(req, newNs); err != nil { + return utils.ErroredResponse(err) + } + + if len(newNs.OwnerReferences) == 0 { + response := admission.Errored(http.StatusBadRequest, fmt.Errorf("the OwnerReference cannot be removed")) + + return &response + } + + if oldNs.GetOwnerReferences()[0].UID != newNs.GetOwnerReferences()[0].UID { + response := admission.Errored(http.StatusBadRequest, fmt.Errorf("the OwnerReference cannot be changed")) + + return &response + } + + return nil + } +} diff --git a/pkg/webhook/namespace/patch.go b/pkg/webhook/namespace/patch.go index 47bc0c3e..13a289e5 100644 --- a/pkg/webhook/namespace/patch.go +++ b/pkg/webhook/namespace/patch.go @@ -1,5 +1,6 @@ // Copyright 2020-2021 Clastix Labs // SPDX-License-Identifier: Apache-2.0 + package namespace import ( diff --git a/pkg/webhook/namespace/user_metadata.go b/pkg/webhook/namespace/user_metadata.go index d3ea4195..ac34be46 100644 --- a/pkg/webhook/namespace/user_metadata.go +++ b/pkg/webhook/namespace/user_metadata.go @@ -112,6 +112,25 @@ func (r *userMetadataHandler) OnUpdate(client client.Client, decoder *admission. } } + if len(tnt.Spec.NodeSelector) > 0 { + v, ok := newNs.GetAnnotations()["scheduler.alpha.kubernetes.io/node-selector"] + if !ok { + response := admission.Denied("the node-selector annotation is enforced, cannot be removed") + + recorder.Eventf(tnt, corev1.EventTypeWarning, "ForbiddenNodeSelectorDeletion", string(response.Result.Reason)) + + return &response + } + + if v != oldNs.GetAnnotations()["scheduler.alpha.kubernetes.io/node-selector"] { + response := admission.Denied("the the node-selector annotation is enforced, cannot be updated") + + recorder.Eventf(tnt, corev1.EventTypeWarning, "ForbiddenNodeSelectorUpdate", string(response.Result.Reason)) + + return &response + } + } + var labels, annotations map[string]string for key, value := range newNs.GetLabels() { diff --git a/pkg/webhook/ownerreference/patching.go b/pkg/webhook/ownerreference/patching.go index ac465ebd..df7f963a 100644 --- a/pkg/webhook/ownerreference/patching.go +++ b/pkg/webhook/ownerreference/patching.go @@ -49,7 +49,7 @@ func (h *handler) OnDelete(client client.Client, decoder *admission.Decoder, rec func (h *handler) OnUpdate(client client.Client, decoder *admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.setOwnerRef(ctx, req, client, decoder, recorder) + return nil } } diff --git a/pkg/webhook/tenant/cordoning.go b/pkg/webhook/tenant/cordoning.go index f1b4ebd1..cc024ac7 100644 --- a/pkg/webhook/tenant/cordoning.go +++ b/pkg/webhook/tenant/cordoning.go @@ -44,7 +44,7 @@ func (h *cordoningHandler) cordonHandler(ctx context.Context, clt client.Client, } tnt := tntList.Items[0] - if tnt.IsCordoned() && utils.IsCapsuleUser(req, h.configuration.UserGroups()) { + if tnt.IsCordoned() && utils.IsCapsuleUser(ctx, req, clt, h.configuration.UserGroups()) { recorder.Eventf(&tnt, corev1.EventTypeWarning, "TenantFreezed", "%s %s/%s cannot be %sd, current Tenant is freezed", req.Kind.String(), req.Namespace, req.Name, strings.ToLower(string(req.Operation))) response := admission.Denied(fmt.Sprintf("tenant %s is freezed: please, reach out to the system administrator", tnt.GetName())) diff --git a/pkg/webhook/utils/in_capsule_groups.go b/pkg/webhook/utils/in_capsule_groups.go index 6278e0b7..9089cec7 100644 --- a/pkg/webhook/utils/in_capsule_groups.go +++ b/pkg/webhook/utils/in_capsule_groups.go @@ -28,7 +28,7 @@ type handler struct { func (h *handler) OnCreate(client client.Client, decoder *admission.Decoder, recorder record.EventRecorder) webhook.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - if !IsCapsuleUser(req, h.configuration.UserGroups()) { + if !IsCapsuleUser(ctx, req, client, h.configuration.UserGroups()) { return nil } @@ -44,7 +44,7 @@ func (h *handler) OnCreate(client client.Client, decoder *admission.Decoder, rec func (h *handler) OnDelete(client client.Client, decoder *admission.Decoder, recorder record.EventRecorder) webhook.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - if !IsCapsuleUser(req, h.configuration.UserGroups()) { + if !IsCapsuleUser(ctx, req, client, h.configuration.UserGroups()) { return nil } @@ -60,7 +60,7 @@ func (h *handler) OnDelete(client client.Client, decoder *admission.Decoder, rec func (h *handler) OnUpdate(client client.Client, decoder *admission.Decoder, recorder record.EventRecorder) webhook.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - if !IsCapsuleUser(req, h.configuration.UserGroups()) { + if !IsCapsuleUser(ctx, req, client, h.configuration.UserGroups()) { return nil } diff --git a/pkg/webhook/utils/is_capsule_user.go b/pkg/webhook/utils/is_capsule_user.go index 0acb4293..5c163073 100644 --- a/pkg/webhook/utils/is_capsule_user.go +++ b/pkg/webhook/utils/is_capsule_user.go @@ -1,12 +1,19 @@ package utils import ( + "context" + "strings" + + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + capsulev1beta1 "github.com/clastix/capsule/api/v1beta1" "github.com/clastix/capsule/pkg/utils" ) -func IsCapsuleUser(req admission.Request, userGroups []string) bool { +func IsCapsuleUser(ctx context.Context, req admission.Request, clt client.Client, userGroups []string) bool { groupList := utils.NewUserGroupList(req.UserInfo.Groups) // if the user is a ServiceAccount belonging to the kube-system namespace, definitely, it's not a Capsule user // and we can skip the check in case of Capsule user group assigned to system:authenticated @@ -14,6 +21,23 @@ func IsCapsuleUser(req admission.Request, userGroups []string) bool { if groupList.Find("system:serviceaccounts:kube-system") { return false } + // nolint:nestif + if sets.NewString(req.UserInfo.Groups...).Has("system:serviceaccounts") { + parts := strings.Split(req.UserInfo.Username, ":") + + targetNamespace := parts[2] + + if len(targetNamespace) > 0 { + tl := &capsulev1beta1.TenantList{} + if err := clt.List(ctx, tl, client.MatchingFieldsSelector{Selector: fields.OneTermEqualSelector(".status.namespaces", targetNamespace)}); err != nil { + return false + } + + if len(tl.Items) == 1 { + return true + } + } + } for _, group := range userGroups { if groupList.Find(group) {