Forbidden node labels and annotations (#464)

* feat: forbidden node labels and annotations * test(e2e): forbidden node labels and annotations * build(kustomize): forbidden node labels and annotations * build(helm): forbidden node labels and annotations * build(installer): forbidden node labels and annotations * chore(make): forbidden node labels and annotations * docs: forbidden node labels and annotations * test(e2e): forbidden node labels and annotations. Use EventuallyCreation func * feat: forbidden node labels and annotations. Check kubernetes version * test(e2e): forbidden node labels and annotations. Check kubernetes version * docs: forbidden node labels and annotations. Version restrictions * feat: forbidden node labels and annotations. Do not update deepcopy functions * docs: forbidden node labels and annotations. Use blockquotes for notes Co-authored-by: Maksim Fedotov <[email protected]>
projectcapsule · Nov 2, 2021 · 14f9686 · 14f9686
1 parent 6ba9826
commit 14f9686
Show file tree

Hide file tree

Showing 23 changed files with 680 additions and 45 deletions.
diff --git a/Makefile b/Makefile
@@ -126,7 +126,8 @@ dev-setup:
 			{'op': 'replace', 'path': '/webhooks/4/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/pods\",'caBundle':\"$${CA_BUNDLE}\"}},\
 			{'op': 'replace', 'path': '/webhooks/5/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/persistentvolumeclaims\",'caBundle':\"$${CA_BUNDLE}\"}},\
 			{'op': 'replace', 'path': '/webhooks/6/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/services\",'caBundle':\"$${CA_BUNDLE}\"}},\
-			{'op': 'replace', 'path': '/webhooks/7/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/tenants\",'caBundle':\"$${CA_BUNDLE}\"}}\
+			{'op': 'replace', 'path': '/webhooks/7/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/tenants\",'caBundle':\"$${CA_BUNDLE}\"}},\
+			{'op': 'replace', 'path': '/webhooks/8/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/nodes\",'caBundle':\"$${CA_BUNDLE}\"}}\
 		]";
 
 # Build the docker image

diff --git a/api/v1alpha1/capsuleconfiguration_annotations.go b/api/v1alpha1/capsuleconfiguration_annotations.go
@@ -0,0 +1,8 @@
+package v1alpha1
+
+const (
+	ForbiddenNodeLabelsAnnotation            = "capsule.clastix.io/forbidden-node-labels"
+	ForbiddenNodeLabelsRegexpAnnotation      = "capsule.clastix.io/forbidden-node-labels-regexp"
+	ForbiddenNodeAnnotationsAnnotation       = "capsule.clastix.io/forbidden-node-annotations"
+	ForbiddenNodeAnnotationsRegexpAnnotation = "capsule.clastix.io/forbidden-node-annotations-regexp"
+)
diff --git a/charts/capsule/README.md b/charts/capsule/README.md
@@ -80,6 +80,7 @@ Parameter | Description | Default
 `manager.resources.limits/cpu` | Set the memory limits assigned to the controller. | `128Mi`
 `mutatingWebhooksTimeoutSeconds` | Timeout in seconds for mutating webhooks. | `30`
 `validatingWebhooksTimeoutSeconds` | Timeout in seconds for validating webhooks. | `30`
+`webhooks` | Additional configuration for capsule webhooks. |
 `imagePullSecrets` | Configuration for `imagePullSecrets` so that you can use a private images registry. | `[]`
 `serviceAccount.create` | Specifies whether a service account should be created. | `true`
 `serviceAccount.annotations` | Annotations to add to the service account. | `{}`

diff --git a/charts/capsule/templates/validatingwebhookconfiguration.yaml b/charts/capsule/templates/validatingwebhookconfiguration.yaml
@@ -240,3 +240,29 @@ webhooks:
       scope: '*'
   sideEffects: None
   timeoutSeconds: {{ .Values.validatingWebhooksTimeoutSeconds }}
+- admissionReviewVersions:
+    - v1
+    - v1beta1
+  clientConfig:
+    caBundle: Cg==
+    service:
+      name: {{ include "capsule.fullname" . }}-webhook-service
+      namespace: {{ .Release.Namespace }}
+      path: /nodes
+      port: 443
+  failurePolicy: {{ .Values.webhooks.nodes.failurePolicy }}
+  name: nodes.capsule.clastix.io
+  matchPolicy: Exact
+  namespaceSelector: {}
+  objectSelector: {}
+  rules:
+    - apiGroups:
+        - ""
+      apiVersions:
+        - v1
+      operations:
+        - UPDATE
+      resources:
+        - nodes
+  sideEffects: None
+  timeoutSeconds: {{ .Values.validatingWebhooksTimeoutSeconds }}
diff --git a/charts/capsule/values.yaml b/charts/capsule/values.yaml
@@ -123,5 +123,7 @@ webhooks:
       matchExpressions:
         - key: capsule.clastix.io/tenant
           operator: Exists
+  nodes:
+    failurePolicy: Fail
 mutatingWebhooksTimeoutSeconds: 30
 validatingWebhooksTimeoutSeconds: 30
diff --git a/config/install.yaml b/config/install.yaml
@@ -1588,14 +1588,33 @@ webhooks:
     service:
       name: capsule-webhook-service
       namespace: capsule-system
-      path: /pods
+      path: /nodes
   failurePolicy: Fail
-  name: pods.capsule.clastix.io
+  name: nodes.capsule.clastix.io
   namespaceSelector:
     matchExpressions:
     - key: capsule.clastix.io/tenant
       operator: Exists
   rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - UPDATE
+    resources:
+    - nodes
+  sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: capsule-webhook-service
+      namespace: capsule-system
+      path: /pods
+  failurePolicy: Fail
+  name: pods.capsule.clastix.io
+  rules:
   - apiGroups:
     - ""
     apiVersions:

diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml
@@ -118,6 +118,25 @@ webhooks:
     resources:
     - networkpolicies
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /nodes
+  failurePolicy: Fail
+  name: nodes.capsule.clastix.io
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - UPDATE
+    resources:
+    - nodes
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:

diff --git a/config/webhook/patch_ns_selector.yaml b/config/webhook/patch_ns_selector.yaml
@@ -23,13 +23,13 @@
       - key: capsule.clastix.io/tenant
         operator: Exists
 - op: add
-  path: /webhooks/5/namespaceSelector
+  path: /webhooks/6/namespaceSelector
   value:
     matchExpressions:
       - key: capsule.clastix.io/tenant
         operator: Exists
 - op: add
-  path: /webhooks/6/namespaceSelector
+  path: /webhooks/7/namespaceSelector
   value:
     matchExpressions:
       - key: capsule.clastix.io/tenant
@@ -43,12 +43,12 @@
 - op: add
   path: /webhooks/3/rules/0/scope
   value: Namespaced
-- op: add
-  path: /webhooks/4/rules/0/scope
-  value: Namespaced
 - op: add
   path: /webhooks/5/rules/0/scope
   value: Namespaced
 - op: add
   path: /webhooks/6/rules/0/scope
   value: Namespaced
+- op: add
+  path: /webhooks/7/rules/0/scope
+  value: Namespaced
diff --git a/controllers/servicelabels/endpoint_slices.go b/controllers/servicelabels/endpoint_slices.go
@@ -14,8 +14,8 @@ type EndpointSlicesLabelsReconciler struct {
 	abstractServiceLabelsReconciler
 
 	Log          logr.Logger
-	VersionMinor int
-	VersionMajor int
+	VersionMinor uint
+	VersionMajor uint
 }
 
 func (r *EndpointSlicesLabelsReconciler) SetupWithManager(mgr ctrl.Manager) error {

diff --git a/docs/operator/use-cases/deny-wildcard-hostnames.md b/docs/operator/use-cases/deny-wildcard-hostnames.md
@@ -26,7 +26,4 @@ EOF
 Doing this, Alice will not be able to use `oil.bigorg.com`, being the tenant-owner of `gas`.
 
 # What’s next
-
-This ends our tour in Capsule use cases. As we improve Capsule, more  use cases about multi-tenancy, policy admission control, and cluster  governance will be covered in the future.
-
-Stay tuned!
+See how Bill, the cluster admin can protect specific labels and annotations on Nodes from modifications by Tenant Owners. [Denying specific user-defined labels or annotations on Nodes](./node-labels-and-annotations.md).
diff --git a/docs/operator/use-cases/namespace-labels-and-annotations.md b/docs/operator/use-cases/namespace-labels-and-annotations.md
@@ -1,4 +1,4 @@
-# Denying user-defined labels or annotations
+# Denying specific user-defined labels or annotations on Namespaces
 
 By default, capsule allows tenant owners to add and modify any label or annotation on their namespaces. 
 
@@ -13,9 +13,9 @@ kind: Tenant
 metadata:
   name: oil
   annotations:
-    capsule.clastix.io/forbidden-namespace-labels: foo.acme.net, bar.acme.net
+    capsule.clastix.io/forbidden-namespace-labels: foo.acme.net,bar.acme.net
     capsule.clastix.io/forbidden-namespace-labels-regexp: .*.acme.net
-    capsule.clastix.io/forbidden-namespace-annotations: foo.acme.net, bar.acme.net
+    capsule.clastix.io/forbidden-namespace-annotations: foo.acme.net,bar.acme.net
     capsule.clastix.io/forbidden-namespace-annotations-regexp: .*.acme.net
 spec:
   owners:

diff --git a/docs/operator/use-cases/node-labels-and-annotations.md b/docs/operator/use-cases/node-labels-and-annotations.md
@@ -0,0 +1,41 @@
+# Denying specific user-defined labels or annotations on Nodes
+
+When using `capsule` together with [capsule-proxy](https://github.com/clastix/capsule-proxy), Bill can allow Tenant Owners to [modify Nodes](../../proxy/overview.md).
+
+By default, it will allow tenant owners to add and modify any label or annotation on their nodes. 
+
+But there are some scenarios, when tenant owners should not have an ability to add or modify specific labels or annotations (there are some types of labels or annotations, which must be protected from modifications - for example, which are set by `cloud-providers` or `autoscalers`).
+
+Bill, the cluster admin, can deny Tenant Owners to add or modify specific labels and annotations on Nodes:
+
+```yaml
+kubectl apply -f - << EOF
+apiVersion: capsule.clastix.io/v1alpha1
+kind: CapsuleConfiguration
+metadata:
+  name: default
+  annotations:
+    capsule.clastix.io/forbidden-node-labels: foo.acme.net,bar.acme.net
+    capsule.clastix.io/forbidden-node-labels-regexp: .*.acme.net
+    capsule.clastix.io/forbidden-node-annotations: foo.acme.net,bar.acme.net
+    capsule.clastix.io/forbidden-node-annotations-regexp: .*.acme.net
+spec:
+  userGroups:
+    - capsule.clastix.io
+    - system:serviceaccounts:default
+EOF
+```
+
+> **Important note**
+>
+>Due to [CVE-2021-25735](https://github.com/kubernetes/kubernetes/issues/100096) this feature is only supported for Kubernetes version older than:
+>* v1.18.18
+>* v1.19.10
+>* v1.20.6
+>* v1.21.0
+
+# What’s next
+
+This ends our tour in Capsule use cases. As we improve Capsule, more  use cases about multi-tenancy, policy admission control, and cluster  governance will be covered in the future.
+
+Stay tuned!
diff --git a/docs/operator/use-cases/taint-services.md b/docs/operator/use-cases/taint-services.md
@@ -25,4 +25,4 @@ EOF
 When Alice creates a service in a namespace, this will inherit the given label and/or annotation.
 
 # What’s next
-See how Bill, the cluster admin, can allow Alice to use specific labels or annotations. [Allow adding labels and annotations on namespaces](./namespace-labels-and-annotations.md).
+See how Bill, the cluster admin, can protect specific labels and annotations on Namespaces from modifications by Alice. [Denying specific user-defined labels or annotations on Namespaces](./namespace-labels-and-annotations.md).
diff --git a/e2e/ingress_class_extensions_test.go b/e2e/ingress_class_extensions_test.go
@@ -184,8 +184,8 @@ var _ = Describe("when Tenant handles Ingress classes with extensions/v1beta1",
 			}
 		}
 
-		if maj, min, v := GetKubernetesSemVer(); maj == 1 && min < 18 {
-			Skip("Running test on Kubernetes " + v + ", doesn't provide .spec.ingressClassName")
+		if version := GetKubernetesVersion(); version.Major() == 1 && version.Minor() < 18 {
+			Skip("Running test on Kubernetes " + version.String() + ", doesn't provide .spec.ingressClassName")
 		}
 
 		ns := NewNamespace("ingress-class-allowed-annotation-extensions-v1beta1")
@@ -265,8 +265,8 @@ var _ = Describe("when Tenant handles Ingress classes with extensions/v1beta1",
 				}
 			}
 
-			if maj, min, v := GetKubernetesSemVer(); maj == 1 && min < 18 {
-				Skip("Running test on Kubernetes " + v + ", doesn't provide .spec.ingressClassName")
+			if version := GetKubernetesVersion(); version.Major() == 1 && version.Minor() < 18 {
+				Skip("Running test on Kubernetes " + version.String() + ", doesn't provide .spec.ingressClassName")
 			}
 
 			i := &extensionsv1beta1.Ingress{