Skip to content

Commit

Permalink
fix: preventing serviceaccount privilege escalation
Browse files Browse the repository at this point in the history
  • Loading branch information
prometherion committed Dec 2, 2022
1 parent 132ffd5 commit 75525ac
Show file tree
Hide file tree
Showing 11 changed files with 154 additions and 14 deletions.
7 changes: 2 additions & 5 deletions controllers/tenant/namespaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

capsulev1beta1 "github.com/clastix/capsule/api/v1beta1"
"github.com/clastix/capsule/pkg/utils"
)

// Ensuring all annotations are applied to each Namespace handled by the Tenant.
Expand Down Expand Up @@ -72,11 +73,7 @@ func (r *Manager) syncNamespaceMetadata(ctx context.Context, namespace string, t
}

if tnt.Spec.NodeSelector != nil {
var selector []string
for k, v := range tnt.Spec.NodeSelector {
selector = append(selector, fmt.Sprintf("%s=%s", k, v))
}
annotations["scheduler.alpha.kubernetes.io/node-selector"] = strings.Join(selector, ",")
annotations = utils.BuildNodeSelector(tnt, annotations)
}

if tnt.Spec.IngressOptions.AllowedClasses != nil {
Expand Down
2 changes: 1 addition & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ func main() {
route.Service(service.Handler()),
route.NetworkPolicy(utils.InCapsuleGroups(cfg, networkpolicy.Handler())),
route.Tenant(tenant.NameHandler(), tenant.RoleBindingRegexHandler(), tenant.IngressClassRegexHandler(), tenant.StorageClassRegexHandler(), tenant.ContainerRegistryRegexHandler(), tenant.HostnameRegexHandler(), tenant.FreezedEmitter(), tenant.ServiceAccountNameHandler(), tenant.ForbiddenAnnotationsRegexHandler(), tenant.ProtectedHandler()),
route.OwnerReference(utils.InCapsuleGroups(cfg, ownerreference.Handler(cfg))),
route.OwnerReference(utils.InCapsuleGroups(cfg, namespacewebhook.OwnerReferenceHandler(), ownerreference.Handler(cfg))),
route.Cordoning(tenant.CordoningHandler(cfg), tenant.ResourceCounterHandler()),
route.Node(utils.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))),
)
Expand Down
35 changes: 35 additions & 0 deletions pkg/utils/node_selector.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// Copyright 2020-2021 Clastix Labs
// SPDX-License-Identifier: Apache-2.0

package utils

import (
"fmt"
"sort"
"strings"

capsulev1beta1 "github.com/clastix/capsule/api/v1beta1"
)

const (
NodeSelectorAnnotation = "scheduler.alpha.kubernetes.io/node-selector"
)

func BuildNodeSelector(tnt *capsulev1beta1.Tenant, nsAnnotations map[string]string) map[string]string {
if nsAnnotations == nil {
nsAnnotations = make(map[string]string)
}

selector := make([]string, 0, len(tnt.Spec.NodeSelector))

for k, v := range tnt.Spec.NodeSelector {
selector = append(selector, fmt.Sprintf("%s=%s", k, v))
}
// Sorting the resulting slice: iterating over maps is randomized, and we could end-up
// in multiple reconciliations upon multiple node selectors.
sort.Strings(selector)

nsAnnotations[NodeSelectorAnnotation] = strings.Join(selector, ",")

return nsAnnotations
}
4 changes: 2 additions & 2 deletions pkg/webhook/namespace/freezed.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func (r *freezedHandler) OnDelete(c client.Client, _ *admission.Decoder, recorde

tnt := tntList.Items[0]

if tnt.IsCordoned() && utils.IsCapsuleUser(req, r.configuration.UserGroups()) {
if tnt.IsCordoned() && utils.IsCapsuleUser(ctx, req, c, r.configuration.UserGroups()) {
recorder.Eventf(&tnt, corev1.EventTypeWarning, "TenantFreezed", "Namespace %s cannot be deleted, the current Tenant is freezed", req.Name)

response := admission.Denied("the selected Tenant is freezed")
Expand Down Expand Up @@ -101,7 +101,7 @@ func (r *freezedHandler) OnUpdate(c client.Client, decoder *admission.Decoder, r

tnt := tntList.Items[0]

if tnt.IsCordoned() && utils.IsCapsuleUser(req, r.configuration.UserGroups()) {
if tnt.IsCordoned() && utils.IsCapsuleUser(ctx, req, c, r.configuration.UserGroups()) {
recorder.Eventf(&tnt, corev1.EventTypeWarning, "TenantFreezed", "Namespace %s cannot be updated, the current Tenant is freezed", ns.GetName())

response := admission.Denied("the selected Tenant is freezed")
Expand Down
64 changes: 64 additions & 0 deletions pkg/webhook/namespace/owner_reference.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// Copyright 2020-2021 Clastix Labs
// SPDX-License-Identifier: Apache-2.0

package namespace

import (
"context"
"fmt"
"net/http"

corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"

capsulewebhook "github.com/clastix/capsule/pkg/webhook"
"github.com/clastix/capsule/pkg/webhook/utils"
)

type ownerReferenceHandler struct{}

func OwnerReferenceHandler() capsulewebhook.Handler {
return &ownerReferenceHandler{}
}

func (r *ownerReferenceHandler) OnCreate(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func {
return func(ctx context.Context, req admission.Request) *admission.Response {
return nil
}
}

func (r *ownerReferenceHandler) OnDelete(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func {
return func(ctx context.Context, req admission.Request) *admission.Response {
return nil
}
}

func (r *ownerReferenceHandler) OnUpdate(_ client.Client, decoder *admission.Decoder, _ record.EventRecorder) capsulewebhook.Func {
return func(ctx context.Context, req admission.Request) *admission.Response {
oldNs := &corev1.Namespace{}
if err := decoder.DecodeRaw(req.OldObject, oldNs); err != nil {
return utils.ErroredResponse(err)
}

newNs := &corev1.Namespace{}
if err := decoder.Decode(req, newNs); err != nil {
return utils.ErroredResponse(err)
}

if len(newNs.OwnerReferences) == 0 {
response := admission.Errored(http.StatusBadRequest, fmt.Errorf("the OwnerReference cannot be removed"))

return &response
}

if oldNs.GetOwnerReferences()[0].UID != newNs.GetOwnerReferences()[0].UID {
response := admission.Errored(http.StatusBadRequest, fmt.Errorf("the OwnerReference cannot be changed"))

return &response
}

return nil
}
}
1 change: 1 addition & 0 deletions pkg/webhook/namespace/patch.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// Copyright 2020-2021 Clastix Labs
// SPDX-License-Identifier: Apache-2.0

package namespace

import (
Expand Down
19 changes: 19 additions & 0 deletions pkg/webhook/namespace/user_metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,25 @@ func (r *userMetadataHandler) OnUpdate(client client.Client, decoder *admission.
}
}

if len(tnt.Spec.NodeSelector) > 0 {
v, ok := newNs.GetAnnotations()["scheduler.alpha.kubernetes.io/node-selector"]
if !ok {
response := admission.Denied("the node-selector annotation is enforced, cannot be removed")

recorder.Eventf(tnt, corev1.EventTypeWarning, "ForbiddenNodeSelectorDeletion", string(response.Result.Reason))

return &response
}

if v != oldNs.GetAnnotations()["scheduler.alpha.kubernetes.io/node-selector"] {
response := admission.Denied("the the node-selector annotation is enforced, cannot be updated")

recorder.Eventf(tnt, corev1.EventTypeWarning, "ForbiddenNodeSelectorUpdate", string(response.Result.Reason))

return &response
}
}

var labels, annotations map[string]string

for key, value := range newNs.GetLabels() {
Expand Down
2 changes: 1 addition & 1 deletion pkg/webhook/ownerreference/patching.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func (h *handler) OnDelete(client client.Client, decoder *admission.Decoder, rec

func (h *handler) OnUpdate(client client.Client, decoder *admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func {
return func(ctx context.Context, req admission.Request) *admission.Response {
return h.setOwnerRef(ctx, req, client, decoder, recorder)
return nil
}
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/webhook/tenant/cordoning.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func (h *cordoningHandler) cordonHandler(ctx context.Context, clt client.Client,
}

tnt := tntList.Items[0]
if tnt.IsCordoned() && utils.IsCapsuleUser(req, h.configuration.UserGroups()) {
if tnt.IsCordoned() && utils.IsCapsuleUser(ctx, req, clt, h.configuration.UserGroups()) {
recorder.Eventf(&tnt, corev1.EventTypeWarning, "TenantFreezed", "%s %s/%s cannot be %sd, current Tenant is freezed", req.Kind.String(), req.Namespace, req.Name, strings.ToLower(string(req.Operation)))

response := admission.Denied(fmt.Sprintf("tenant %s is freezed: please, reach out to the system administrator", tnt.GetName()))
Expand Down
6 changes: 3 additions & 3 deletions pkg/webhook/utils/in_capsule_groups.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ type handler struct {

func (h *handler) OnCreate(client client.Client, decoder *admission.Decoder, recorder record.EventRecorder) webhook.Func {
return func(ctx context.Context, req admission.Request) *admission.Response {
if !IsCapsuleUser(req, h.configuration.UserGroups()) {
if !IsCapsuleUser(ctx, req, client, h.configuration.UserGroups()) {
return nil
}

Expand All @@ -44,7 +44,7 @@ func (h *handler) OnCreate(client client.Client, decoder *admission.Decoder, rec

func (h *handler) OnDelete(client client.Client, decoder *admission.Decoder, recorder record.EventRecorder) webhook.Func {
return func(ctx context.Context, req admission.Request) *admission.Response {
if !IsCapsuleUser(req, h.configuration.UserGroups()) {
if !IsCapsuleUser(ctx, req, client, h.configuration.UserGroups()) {
return nil
}

Expand All @@ -60,7 +60,7 @@ func (h *handler) OnDelete(client client.Client, decoder *admission.Decoder, rec

func (h *handler) OnUpdate(client client.Client, decoder *admission.Decoder, recorder record.EventRecorder) webhook.Func {
return func(ctx context.Context, req admission.Request) *admission.Response {
if !IsCapsuleUser(req, h.configuration.UserGroups()) {
if !IsCapsuleUser(ctx, req, client, h.configuration.UserGroups()) {
return nil
}

Expand Down
26 changes: 25 additions & 1 deletion pkg/webhook/utils/is_capsule_user.go
Original file line number Diff line number Diff line change
@@ -1,19 +1,43 @@
package utils

import (
"context"
"strings"

"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/util/sets"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"

capsulev1beta1 "github.com/clastix/capsule/api/v1beta1"
"github.com/clastix/capsule/pkg/utils"
)

func IsCapsuleUser(req admission.Request, userGroups []string) bool {
func IsCapsuleUser(ctx context.Context, req admission.Request, clt client.Client, userGroups []string) bool {
groupList := utils.NewUserGroupList(req.UserInfo.Groups)
// if the user is a ServiceAccount belonging to the kube-system namespace, definitely, it's not a Capsule user
// and we can skip the check in case of Capsule user group assigned to system:authenticated
// (ref: https://github.com/clastix/capsule/issues/234)
if groupList.Find("system:serviceaccounts:kube-system") {
return false
}
// nolint:nestif
if sets.NewString(req.UserInfo.Groups...).Has("system:serviceaccounts") {
parts := strings.Split(req.UserInfo.Username, ":")

targetNamespace := parts[2]

if len(targetNamespace) > 0 {
tl := &capsulev1beta1.TenantList{}
if err := clt.List(ctx, tl, client.MatchingFieldsSelector{Selector: fields.OneTermEqualSelector(".status.namespaces", targetNamespace)}); err != nil {
return false
}

if len(tl.Items) == 1 {
return true
}
}
}

for _, group := range userGroups {
if groupList.Find(group) {
Expand Down

0 comments on commit 75525ac

Please sign in to comment.