Skip to content

Commit

Permalink
Forbidden node labels and annotations (projectcapsule#464)
Browse files Browse the repository at this point in the history
* feat: forbidden node labels and annotations

* test(e2e): forbidden node labels and annotations

* build(kustomize): forbidden node labels and annotations

* build(helm): forbidden node labels and annotations

* build(installer): forbidden node labels and annotations

* chore(make): forbidden node labels and annotations

* docs: forbidden node labels and annotations

* test(e2e): forbidden node labels and annotations. Use EventuallyCreation func

* feat: forbidden node labels and annotations. Check kubernetes version

* test(e2e): forbidden node labels and annotations. Check kubernetes version

* docs: forbidden node labels and annotations. Version restrictions

* feat: forbidden node labels and annotations. Do not update deepcopy functions

* docs: forbidden node labels and annotations. Use blockquotes for notes

Co-authored-by: Maksim Fedotov <[email protected]>
  • Loading branch information
unai-ttxu and Maksim Fedotov committed Jun 3, 2022
1 parent e44d310 commit 6fa5ec7
Show file tree
Hide file tree
Showing 23 changed files with 862 additions and 40 deletions.
54 changes: 53 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ manager: generate fmt vet

# Run against the configured Kubernetes cluster in ~/.kube/config
run: generate manifests
go run ./main.go
go run .

# Creates the single file to install Capsule without any external dependency
installer: manifests kustomize
Expand Down Expand Up @@ -78,6 +78,58 @@ manifests: controller-gen
generate: controller-gen
$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."

# Setup development env
# Usage:
# LAPTOP_HOST_IP=<YOUR_LAPTOP_IP> make dev-setup
# For example:
# LAPTOP_HOST_IP=192.168.10.101 make dev-setup
define TLS_CNF
[ req ]
default_bits = 4096
distinguished_name = req_distinguished_name
req_extensions = req_ext
[ req_distinguished_name ]
countryName = SG
stateOrProvinceName = SG
localityName = SG
organizationName = CAPSULE
commonName = CAPSULE
[ req_ext ]
subjectAltName = @alt_names
[alt_names]
IP.1 = $(LAPTOP_HOST_IP)
endef
export TLS_CNF
dev-setup:
kubectl -n capsule-system scale deployment capsule-controller-manager --replicas=0
mkdir -p /tmp/k8s-webhook-server/serving-certs
echo "$${TLS_CNF}" > _tls.cnf
openssl req -newkey rsa:4096 -days 3650 -nodes -x509 \
-subj "/C=SG/ST=SG/L=SG/O=CAPSULE/CN=CAPSULE" \
-extensions req_ext \
-config _tls.cnf \
-keyout /tmp/k8s-webhook-server/serving-certs/tls.key \
-out /tmp/k8s-webhook-server/serving-certs/tls.crt
rm -f _tls.cnf
export WEBHOOK_URL="https://$${LAPTOP_HOST_IP}:9443"; \
export CA_BUNDLE=`openssl base64 -in /tmp/k8s-webhook-server/serving-certs/tls.crt | tr -d '\n'`; \
kubectl patch MutatingWebhookConfiguration capsule-mutating-webhook-configuration \
--type='json' -p="[\
{'op': 'replace', 'path': '/webhooks/0/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/mutate-v1-namespace-owner-reference\",'caBundle':\"$${CA_BUNDLE}\"}}\
]" && \
kubectl patch ValidatingWebhookConfiguration capsule-validating-webhook-configuration \
--type='json' -p="[\
{'op': 'replace', 'path': '/webhooks/0/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/cordoning\",'caBundle':\"$${CA_BUNDLE}\"}},\
{'op': 'replace', 'path': '/webhooks/1/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/ingresses\",'caBundle':\"$${CA_BUNDLE}\"}},\
{'op': 'replace', 'path': '/webhooks/2/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/namespaces\",'caBundle':\"$${CA_BUNDLE}\"}},\
{'op': 'replace', 'path': '/webhooks/3/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/networkpolicies\",'caBundle':\"$${CA_BUNDLE}\"}},\
{'op': 'replace', 'path': '/webhooks/4/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/pods\",'caBundle':\"$${CA_BUNDLE}\"}},\
{'op': 'replace', 'path': '/webhooks/5/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/persistentvolumeclaims\",'caBundle':\"$${CA_BUNDLE}\"}},\
{'op': 'replace', 'path': '/webhooks/6/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/services\",'caBundle':\"$${CA_BUNDLE}\"}},\
{'op': 'replace', 'path': '/webhooks/7/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/tenants\",'caBundle':\"$${CA_BUNDLE}\"}},\
{'op': 'replace', 'path': '/webhooks/8/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/nodes\",'caBundle':\"$${CA_BUNDLE}\"}}\
]";

# Build the docker image
docker-build: test
docker build . -t ${IMG} --build-arg GIT_HEAD_COMMIT=$(GIT_HEAD_COMMIT) \
Expand Down
8 changes: 8 additions & 0 deletions api/v1alpha1/capsuleconfiguration_annotations.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package v1alpha1

const (
ForbiddenNodeLabelsAnnotation = "capsule.clastix.io/forbidden-node-labels"
ForbiddenNodeLabelsRegexpAnnotation = "capsule.clastix.io/forbidden-node-labels-regexp"
ForbiddenNodeAnnotationsAnnotation = "capsule.clastix.io/forbidden-node-annotations"
ForbiddenNodeAnnotationsRegexpAnnotation = "capsule.clastix.io/forbidden-node-annotations-regexp"
)
33 changes: 33 additions & 0 deletions api/v1beta1/forbidden_list.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
// Copyright 2020-2021 Clastix Labs
// SPDX-License-Identifier: Apache-2.0
//nolint:dupl
package v1beta1

import (
"regexp"
"sort"
"strings"
)

type ForbiddenListSpec struct {
Exact []string `json:"denied,omitempty"`
Regex string `json:"deniedRegex,omitempty"`
}

func (in *ForbiddenListSpec) ExactMatch(value string) (ok bool) {
if len(in.Exact) > 0 {
sort.SliceStable(in.Exact, func(i, j int) bool {
return strings.ToLower(in.Exact[i]) < strings.ToLower(in.Exact[j])
})
i := sort.SearchStrings(in.Exact, value)
ok = i < len(in.Exact) && in.Exact[i] == value
}
return
}

func (in ForbiddenListSpec) RegexMatch(value string) (ok bool) {
if len(in.Regex) > 0 {
ok = regexp.MustCompile(in.Regex).MatchString(value)
}
return
}
67 changes: 67 additions & 0 deletions api/v1beta1/forbidden_list_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Copyright 2020-2021 Clastix Labs
// SPDX-License-Identifier: Apache-2.0
//nolint:dupl
package v1beta1

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestForbiddenListSpec_ExactMatch(t *testing.T) {
type tc struct {
In []string
True []string
False []string
}
for _, tc := range []tc{
{
[]string{"foo", "bar", "bizz", "buzz"},
[]string{"foo", "bar", "bizz", "buzz"},
[]string{"bing", "bong"},
},
{
[]string{"one", "two", "three"},
[]string{"one", "two", "three"},
[]string{"a", "b", "c"},
},
{
nil,
nil,
[]string{"any", "value"},
},
} {
a := ForbiddenListSpec{
Exact: tc.In,
}
for _, ok := range tc.True {
assert.True(t, a.ExactMatch(ok))
}
for _, ko := range tc.False {
assert.False(t, a.ExactMatch(ko))
}
}
}

func TestForbiddenListSpec_RegexMatch(t *testing.T) {
type tc struct {
Regex string
True []string
False []string
}
for _, tc := range []tc{
{`first-\w+-pattern`, []string{"first-date-pattern", "first-year-pattern"}, []string{"broken", "first-year", "second-date-pattern"}},
{``, nil, []string{"any", "value"}},
} {
a := ForbiddenListSpec{
Regex: tc.Regex,
}
for _, ok := range tc.True {
assert.True(t, a.RegexMatch(ok))
}
for _, ko := range tc.False {
assert.False(t, a.RegexMatch(ko))
}
}
}
1 change: 1 addition & 0 deletions charts/capsule/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ Parameter | Description | Default
`manager.resources.limits/cpu` | Set the memory limits assigned to the controller. | `128Mi`
`mutatingWebhooksTimeoutSeconds` | Timeout in seconds for mutating webhooks. | `30`
`validatingWebhooksTimeoutSeconds` | Timeout in seconds for validating webhooks. | `30`
`webhooks` | Additional configuration for capsule webhooks. |
`imagePullSecrets` | Configuration for `imagePullSecrets` so that you can use a private images registry. | `[]`
`serviceAccount.create` | Specifies whether a service account should be created. | `true`
`serviceAccount.annotations` | Annotations to add to the service account. | `{}`
Expand Down
26 changes: 26 additions & 0 deletions charts/capsule/templates/validatingwebhookconfiguration.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -252,3 +252,29 @@ webhooks:
scope: '*'
sideEffects: None
timeoutSeconds: {{ .Values.validatingWebhooksTimeoutSeconds }}
- admissionReviewVersions:
- v1
- v1beta1
clientConfig:
caBundle: Cg==
service:
name: {{ include "capsule.fullname" . }}-webhook-service
namespace: {{ .Release.Namespace }}
path: /nodes
port: 443
failurePolicy: {{ .Values.webhooks.nodes.failurePolicy }}
name: nodes.capsule.clastix.io
matchPolicy: Exact
namespaceSelector: {}
objectSelector: {}
rules:
- apiGroups:
- ""
apiVersions:
- v1
operations:
- UPDATE
resources:
- nodes
sideEffects: None
timeoutSeconds: {{ .Values.validatingWebhooksTimeoutSeconds }}
51 changes: 49 additions & 2 deletions charts/capsule/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ jobs:
repository: quay.io/clastix/kubectl
pullPolicy: IfNotPresent
tag: "v1.20.7"
mutatingWebhooksTimeoutSeconds: 30
validatingWebhooksTimeoutSeconds: 30
imagePullSecrets: []
serviceAccount:
create: true
Expand Down Expand Up @@ -80,3 +78,52 @@ customLabels: {}

# Additional annotations
customAnnotations: {}

# Webhooks configurations
webhooks:
namespaceOwnerReference:
failurePolicy: Fail
cordoning:
failurePolicy: Fail
namespaceSelector:
matchExpressions:
- key: capsule.clastix.io/tenant
operator: Exists
ingresses:
failurePolicy: Fail
namespaceSelector:
matchExpressions:
- key: capsule.clastix.io/tenant
operator: Exists
namespaces:
failurePolicy: Fail
networkpolicies:
failurePolicy: Fail
namespaceSelector:
matchExpressions:
- key: capsule.clastix.io/tenant
operator: Exists
pods:
failurePolicy: Fail
namespaceSelector:
matchExpressions:
- key: capsule.clastix.io/tenant
operator: Exists
persistentvolumeclaims:
failurePolicy: Fail
namespaceSelector:
matchExpressions:
- key: capsule.clastix.io/tenant
operator: Exists
tenants:
failurePolicy: Fail
services:
failurePolicy: Fail
namespaceSelector:
matchExpressions:
- key: capsule.clastix.io/tenant
operator: Exists
nodes:
failurePolicy: Fail
mutatingWebhooksTimeoutSeconds: 30
validatingWebhooksTimeoutSeconds: 30
23 changes: 21 additions & 2 deletions config/install.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1587,14 +1587,33 @@ webhooks:
service:
name: capsule-webhook-service
namespace: capsule-system
path: /pods
path: /nodes
failurePolicy: Fail
name: pods.capsule.clastix.io
name: nodes.capsule.clastix.io
namespaceSelector:
matchExpressions:
- key: capsule.clastix.io/tenant
operator: Exists
rules:
- apiGroups:
- ""
apiVersions:
- v1
operations:
- UPDATE
resources:
- nodes
sideEffects: None
- admissionReviewVersions:
- v1
clientConfig:
service:
name: capsule-webhook-service
namespace: capsule-system
path: /pods
failurePolicy: Fail
name: pods.capsule.clastix.io
rules:
- apiGroups:
- ""
apiVersions:
Expand Down
19 changes: 19 additions & 0 deletions config/webhook/manifests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,25 @@ webhooks:
resources:
- networkpolicies
sideEffects: None
- admissionReviewVersions:
- v1
clientConfig:
service:
name: webhook-service
namespace: system
path: /nodes
failurePolicy: Fail
name: nodes.capsule.clastix.io
rules:
- apiGroups:
- ""
apiVersions:
- v1
operations:
- UPDATE
resources:
- nodes
sideEffects: None
- admissionReviewVersions:
- v1
clientConfig:
Expand Down
10 changes: 5 additions & 5 deletions config/webhook/patch_ns_selector.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@
- key: capsule.clastix.io/tenant
operator: Exists
- op: add
path: /webhooks/5/namespaceSelector
path: /webhooks/6/namespaceSelector
value:
matchExpressions:
- key: capsule.clastix.io/tenant
operator: Exists
- op: add
path: /webhooks/6/namespaceSelector
path: /webhooks/7/namespaceSelector
value:
matchExpressions:
- key: capsule.clastix.io/tenant
Expand All @@ -43,12 +43,12 @@
- op: add
path: /webhooks/3/rules/0/scope
value: Namespaced
- op: add
path: /webhooks/4/rules/0/scope
value: Namespaced
- op: add
path: /webhooks/5/rules/0/scope
value: Namespaced
- op: add
path: /webhooks/6/rules/0/scope
value: Namespaced
- op: add
path: /webhooks/7/rules/0/scope
value: Namespaced
4 changes: 2 additions & 2 deletions controllers/servicelabels/endpoint_slices.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ type EndpointSlicesLabelsReconciler struct {
abstractServiceLabelsReconciler

Log logr.Logger
VersionMinor int
VersionMajor int
VersionMinor uint
VersionMajor uint
}

func (r *EndpointSlicesLabelsReconciler) SetupWithManager(mgr ctrl.Manager) error {
Expand Down
5 changes: 1 addition & 4 deletions docs/operator/use-cases/deny-wildcard-hostnames.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,4 @@ EOF
Doing this, Alice will not be able to use `oil.bigorg.com`, being the tenant-owner of `gas`.

# What’s next

This ends our tour in Capsule use cases. As we improve Capsule, more use cases about multi-tenancy, policy admission control, and cluster governance will be covered in the future.

Stay tuned!
See how Bill, the cluster admin can protect specific labels and annotations on Nodes from modifications by Tenant Owners. [Denying specific user-defined labels or annotations on Nodes](./node-labels-and-annotations.md).
Loading

0 comments on commit 6fa5ec7

Please sign in to comment.