Forbidden node labels and annotations (projectcapsule#464)

* feat: forbidden node labels and annotations * test(e2e): forbidden node labels and annotations * build(kustomize): forbidden node labels and annotations * build(helm): forbidden node labels and annotations * build(installer): forbidden node labels and annotations * chore(make): forbidden node labels and annotations * docs: forbidden node labels and annotations * test(e2e): forbidden node labels and annotations. Use EventuallyCreation func * feat: forbidden node labels and annotations. Check kubernetes version * test(e2e): forbidden node labels and annotations. Check kubernetes version * docs: forbidden node labels and annotations. Version restrictions * feat: forbidden node labels and annotations. Do not update deepcopy functions * docs: forbidden node labels and annotations. Use blockquotes for notes Co-authored-by: Maksim Fedotov <[email protected]>
unai-ttxu · Jun 3, 2022 · cadcb9e · cadcb9e
1 parent 8e1da08
commit cadcb9e
Show file tree

Hide file tree

Showing 22 changed files with 774 additions and 43 deletions.
diff --git a/Makefile b/Makefile
@@ -45,7 +45,7 @@ manager: generate fmt vet
 
 # Run against the configured Kubernetes cluster in ~/.kube/config
 run: generate manifests
-	go run ./main.go
+	go run .
 
 # Creates the single file to install Capsule without any external dependency
 installer: manifests kustomize
@@ -78,6 +78,58 @@ manifests: controller-gen
 generate: controller-gen
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
+# Setup development env
+# Usage: 
+# 	LAPTOP_HOST_IP=<YOUR_LAPTOP_IP> make dev-setup
+# For example:
+#	LAPTOP_HOST_IP=192.168.10.101 make dev-setup
+define TLS_CNF
+[ req ]
+default_bits       = 4096
+distinguished_name = req_distinguished_name
+req_extensions     = req_ext
+[ req_distinguished_name ]
+countryName                = SG
+stateOrProvinceName        = SG
+localityName               = SG
+organizationName           = CAPSULE
+commonName                 = CAPSULE
+[ req_ext ]
+subjectAltName = @alt_names
+[alt_names]
+IP.1   = $(LAPTOP_HOST_IP)
+endef
+export TLS_CNF
+dev-setup:
+	kubectl -n capsule-system scale deployment capsule-controller-manager --replicas=0
+	mkdir -p /tmp/k8s-webhook-server/serving-certs
+	echo "$${TLS_CNF}" > _tls.cnf
+	openssl req -newkey rsa:4096 -days 3650 -nodes -x509 \
+		-subj "/C=SG/ST=SG/L=SG/O=CAPSULE/CN=CAPSULE" \
+		-extensions req_ext \
+		-config _tls.cnf \
+		-keyout /tmp/k8s-webhook-server/serving-certs/tls.key \
+		-out /tmp/k8s-webhook-server/serving-certs/tls.crt
+	rm -f _tls.cnf 
+	export WEBHOOK_URL="https://$${LAPTOP_HOST_IP}:9443"; \
+	export CA_BUNDLE=`openssl base64 -in /tmp/k8s-webhook-server/serving-certs/tls.crt | tr -d '\n'`; \
+	kubectl patch MutatingWebhookConfiguration capsule-mutating-webhook-configuration \
+		--type='json' -p="[\
+			{'op': 'replace', 'path': '/webhooks/0/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/mutate-v1-namespace-owner-reference\",'caBundle':\"$${CA_BUNDLE}\"}}\
+		]" && \
+	kubectl patch ValidatingWebhookConfiguration capsule-validating-webhook-configuration \
+		--type='json' -p="[\
+			{'op': 'replace', 'path': '/webhooks/0/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/cordoning\",'caBundle':\"$${CA_BUNDLE}\"}},\
+			{'op': 'replace', 'path': '/webhooks/1/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/ingresses\",'caBundle':\"$${CA_BUNDLE}\"}},\
+			{'op': 'replace', 'path': '/webhooks/2/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/namespaces\",'caBundle':\"$${CA_BUNDLE}\"}},\
+			{'op': 'replace', 'path': '/webhooks/3/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/networkpolicies\",'caBundle':\"$${CA_BUNDLE}\"}},\
+			{'op': 'replace', 'path': '/webhooks/4/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/pods\",'caBundle':\"$${CA_BUNDLE}\"}},\
+			{'op': 'replace', 'path': '/webhooks/5/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/persistentvolumeclaims\",'caBundle':\"$${CA_BUNDLE}\"}},\
+			{'op': 'replace', 'path': '/webhooks/6/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/services\",'caBundle':\"$${CA_BUNDLE}\"}},\
+			{'op': 'replace', 'path': '/webhooks/7/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/tenants\",'caBundle':\"$${CA_BUNDLE}\"}},\
+			{'op': 'replace', 'path': '/webhooks/8/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/nodes\",'caBundle':\"$${CA_BUNDLE}\"}}\
+		]";
+
 # Build the docker image
 docker-build: test
 	docker build . -t ${IMG} --build-arg GIT_HEAD_COMMIT=$(GIT_HEAD_COMMIT) \

diff --git a/api/v1alpha1/capsuleconfiguration_annotations.go b/api/v1alpha1/capsuleconfiguration_annotations.go
@@ -0,0 +1,8 @@
+package v1alpha1
+
+const (
+	ForbiddenNodeLabelsAnnotation            = "capsule.clastix.io/forbidden-node-labels"
+	ForbiddenNodeLabelsRegexpAnnotation      = "capsule.clastix.io/forbidden-node-labels-regexp"
+	ForbiddenNodeAnnotationsAnnotation       = "capsule.clastix.io/forbidden-node-annotations"
+	ForbiddenNodeAnnotationsRegexpAnnotation = "capsule.clastix.io/forbidden-node-annotations-regexp"
+)
diff --git a/charts/capsule/README.md b/charts/capsule/README.md
@@ -80,6 +80,7 @@ Parameter | Description | Default
 `manager.resources.limits/cpu` | Set the memory limits assigned to the controller. | `128Mi`
 `mutatingWebhooksTimeoutSeconds` | Timeout in seconds for mutating webhooks. | `30`
 `validatingWebhooksTimeoutSeconds` | Timeout in seconds for validating webhooks. | `30`
+`webhooks` | Additional configuration for capsule webhooks. |
 `imagePullSecrets` | Configuration for `imagePullSecrets` so that you can use a private images registry. | `[]`
 `serviceAccount.create` | Specifies whether a service account should be created. | `true`
 `serviceAccount.annotations` | Annotations to add to the service account. | `{}`

diff --git a/charts/capsule/templates/validatingwebhookconfiguration.yaml b/charts/capsule/templates/validatingwebhookconfiguration.yaml
@@ -252,3 +252,29 @@ webhooks:
       scope: '*'
   sideEffects: None
   timeoutSeconds: {{ .Values.validatingWebhooksTimeoutSeconds }}
+- admissionReviewVersions:
+    - v1
+    - v1beta1
+  clientConfig:
+    caBundle: Cg==
+    service:
+      name: {{ include "capsule.fullname" . }}-webhook-service
+      namespace: {{ .Release.Namespace }}
+      path: /nodes
+      port: 443
+  failurePolicy: {{ .Values.webhooks.nodes.failurePolicy }}
+  name: nodes.capsule.clastix.io
+  matchPolicy: Exact
+  namespaceSelector: {}
+  objectSelector: {}
+  rules:
+    - apiGroups:
+        - ""
+      apiVersions:
+        - v1
+      operations:
+        - UPDATE
+      resources:
+        - nodes
+  sideEffects: None
+  timeoutSeconds: {{ .Values.validatingWebhooksTimeoutSeconds }}
diff --git a/charts/capsule/values.yaml b/charts/capsule/values.yaml
@@ -42,8 +42,6 @@ jobs:
     repository: quay.io/clastix/kubectl
     pullPolicy: IfNotPresent
     tag: "v1.20.7"
-mutatingWebhooksTimeoutSeconds: 30
-validatingWebhooksTimeoutSeconds: 30
 imagePullSecrets: []
 serviceAccount:
   create: true
@@ -80,3 +78,52 @@ customLabels: {}
 
 # Additional annotations
 customAnnotations: {}
+
+# Webhooks configurations
+webhooks:
+  namespaceOwnerReference:
+    failurePolicy: Fail
+  cordoning:
+    failurePolicy: Fail
+    namespaceSelector:
+      matchExpressions:
+        - key: capsule.clastix.io/tenant
+          operator: Exists
+  ingresses:
+    failurePolicy: Fail
+    namespaceSelector:
+      matchExpressions:
+        - key: capsule.clastix.io/tenant
+          operator: Exists
+  namespaces:
+    failurePolicy: Fail
+  networkpolicies:
+    failurePolicy: Fail
+    namespaceSelector:
+      matchExpressions:
+        - key: capsule.clastix.io/tenant
+          operator: Exists
+  pods:
+    failurePolicy: Fail
+    namespaceSelector:
+      matchExpressions:
+        - key: capsule.clastix.io/tenant
+          operator: Exists
+  persistentvolumeclaims:
+    failurePolicy: Fail
+    namespaceSelector:
+      matchExpressions:
+        - key: capsule.clastix.io/tenant
+          operator: Exists
+  tenants:
+    failurePolicy: Fail
+  services:
+    failurePolicy: Fail
+    namespaceSelector:
+      matchExpressions:
+        - key: capsule.clastix.io/tenant
+          operator: Exists
+  nodes:
+    failurePolicy: Fail
+mutatingWebhooksTimeoutSeconds: 30
+validatingWebhooksTimeoutSeconds: 30
diff --git a/config/install.yaml b/config/install.yaml
@@ -1587,14 +1587,33 @@ webhooks:
     service:
       name: capsule-webhook-service
       namespace: capsule-system
-      path: /pods
+      path: /nodes
   failurePolicy: Fail
-  name: pods.capsule.clastix.io
+  name: nodes.capsule.clastix.io
   namespaceSelector:
     matchExpressions:
     - key: capsule.clastix.io/tenant
       operator: Exists
   rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - UPDATE
+    resources:
+    - nodes
+  sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: capsule-webhook-service
+      namespace: capsule-system
+      path: /pods
+  failurePolicy: Fail
+  name: pods.capsule.clastix.io
+  rules:
   - apiGroups:
     - ""
     apiVersions:

diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml
@@ -117,6 +117,25 @@ webhooks:
     resources:
     - networkpolicies
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /nodes
+  failurePolicy: Fail
+  name: nodes.capsule.clastix.io
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - UPDATE
+    resources:
+    - nodes
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:

diff --git a/config/webhook/patch_ns_selector.yaml b/config/webhook/patch_ns_selector.yaml
@@ -23,13 +23,13 @@
       - key: capsule.clastix.io/tenant
         operator: Exists
 - op: add
-  path: /webhooks/5/namespaceSelector
+  path: /webhooks/6/namespaceSelector
   value:
     matchExpressions:
       - key: capsule.clastix.io/tenant
         operator: Exists
 - op: add
-  path: /webhooks/6/namespaceSelector
+  path: /webhooks/7/namespaceSelector
   value:
     matchExpressions:
       - key: capsule.clastix.io/tenant
@@ -43,12 +43,12 @@
 - op: add
   path: /webhooks/3/rules/0/scope
   value: Namespaced
-- op: add
-  path: /webhooks/4/rules/0/scope
-  value: Namespaced
 - op: add
   path: /webhooks/5/rules/0/scope
   value: Namespaced
 - op: add
   path: /webhooks/6/rules/0/scope
   value: Namespaced
+- op: add
+  path: /webhooks/7/rules/0/scope
+  value: Namespaced
diff --git a/controllers/servicelabels/endpoint_slices.go b/controllers/servicelabels/endpoint_slices.go
@@ -14,8 +14,8 @@ type EndpointSlicesLabelsReconciler struct {
 	abstractServiceLabelsReconciler
 
 	Log          logr.Logger
-	VersionMinor int
-	VersionMajor int
+	VersionMinor uint
+	VersionMajor uint
 }
 
 func (r *EndpointSlicesLabelsReconciler) SetupWithManager(mgr ctrl.Manager) error {

diff --git a/docs/operator/use-cases/deny-wildcard-hostnames.md b/docs/operator/use-cases/deny-wildcard-hostnames.md
@@ -26,7 +26,4 @@ EOF
 Doing this, Alice will not be able to use `oil.bigorg.com`, being the tenant-owner of `gas`.
 
 # What’s next
-
-This ends our tour in Capsule use cases. As we improve Capsule, more  use cases about multi-tenancy, policy admission control, and cluster  governance will be covered in the future.
-
-Stay tuned!
+See how Bill, the cluster admin can protect specific labels and annotations on Nodes from modifications by Tenant Owners. [Denying specific user-defined labels or annotations on Nodes](./node-labels-and-annotations.md).
diff --git a/docs/operator/use-cases/namespace-labels-and-annotations.md b/docs/operator/use-cases/namespace-labels-and-annotations.md
@@ -1,4 +1,4 @@
-# Denying user-defined labels or annotations
+# Denying specific user-defined labels or annotations on Namespaces
 
 By default, capsule allows tenant owners to add and modify any label or annotation on their namespaces. 
 
@@ -13,9 +13,9 @@ kind: Tenant
 metadata:
   name: oil
   annotations:
-    capsule.clastix.io/forbidden-namespace-labels: foo.acme.net, bar.acme.net
+    capsule.clastix.io/forbidden-namespace-labels: foo.acme.net,bar.acme.net
     capsule.clastix.io/forbidden-namespace-labels-regexp: .*.acme.net
-    capsule.clastix.io/forbidden-namespace-annotations: foo.acme.net, bar.acme.net
+    capsule.clastix.io/forbidden-namespace-annotations: foo.acme.net,bar.acme.net
     capsule.clastix.io/forbidden-namespace-annotations-regexp: .*.acme.net
 spec:
   owners:

diff --git a/docs/operator/use-cases/node-labels-and-annotations.md b/docs/operator/use-cases/node-labels-and-annotations.md
@@ -0,0 +1,41 @@
+# Denying specific user-defined labels or annotations on Nodes
+
+When using `capsule` together with [capsule-proxy](https://github.com/clastix/capsule-proxy), Bill can allow Tenant Owners to [modify Nodes](../../proxy/overview.md).
+
+By default, it will allow tenant owners to add and modify any label or annotation on their nodes. 
+
+But there are some scenarios, when tenant owners should not have an ability to add or modify specific labels or annotations (there are some types of labels or annotations, which must be protected from modifications - for example, which are set by `cloud-providers` or `autoscalers`).
+
+Bill, the cluster admin, can deny Tenant Owners to add or modify specific labels and annotations on Nodes:
+
+```yaml
+kubectl apply -f - << EOF
+apiVersion: capsule.clastix.io/v1alpha1
+kind: CapsuleConfiguration
+metadata:
+  name: default
+  annotations:
+    capsule.clastix.io/forbidden-node-labels: foo.acme.net,bar.acme.net
+    capsule.clastix.io/forbidden-node-labels-regexp: .*.acme.net
+    capsule.clastix.io/forbidden-node-annotations: foo.acme.net,bar.acme.net
+    capsule.clastix.io/forbidden-node-annotations-regexp: .*.acme.net
+spec:
+  userGroups:
+    - capsule.clastix.io
+    - system:serviceaccounts:default
+EOF
+```
+
+> **Important note**
+>
+>Due to [CVE-2021-25735](https://github.com/kubernetes/kubernetes/issues/100096) this feature is only supported for Kubernetes version older than:
+>* v1.18.18
+>* v1.19.10
+>* v1.20.6
+>* v1.21.0
+
+# What’s next
+
+This ends our tour in Capsule use cases. As we improve Capsule, more  use cases about multi-tenancy, policy admission control, and cluster  governance will be covered in the future.
+
+Stay tuned!
diff --git a/docs/operator/use-cases/taint-services.md b/docs/operator/use-cases/taint-services.md
@@ -25,4 +25,4 @@ EOF
 When Alice creates a service in a namespace, this will inherit the given label and/or annotation.
 
 # What’s next
-See how Bill, the cluster admin, can allow Alice to use specific labels or annotations. [Allow adding labels and annotations on namespaces](./namespace-labels-and-annotations.md).
+See how Bill, the cluster admin, can protect specific labels and annotations on Namespaces from modifications by Alice. [Denying specific user-defined labels or annotations on Namespaces](./namespace-labels-and-annotations.md).