From b7b101809201001f2c04aee4ba0a8c6ef2e04dae Mon Sep 17 00:00:00 2001 From: bsctl Date: Sat, 17 Jul 2021 14:05:51 +0200 Subject: [PATCH] docs: add further test cases --- ...-service-management-of-network-policies.md | 77 +++++++++++ ...self-service-management-of-rolebindings.md | 58 ++++++++ .../allow-self-service-management-of-roles.md | 58 ++++++++ .../block-access-to-multitenant-resources.md | 2 +- .../block-access-to-other-tenant-resources.md | 4 +- docs/operator/mtb/block-add-capabilities.md | 2 +- ...network-access-across-tenant-namespaces.md | 107 +++++++++++++++ .../mtb/block-privilege-escalation.md | 2 +- .../mtb/block-privileged-containers.md | 2 +- ...lock-use-of-existing-persistent-volumes.md | 40 ++++++ docs/operator/mtb/block-use-of-host-ipc.md | 115 ++++++++++++++++ .../block-use-of-host-networking-and-ports.md | 4 +- .../mtb/block-use-of-host-path-volumes.md | 4 +- docs/operator/mtb/block-use-of-host-pid.md | 115 ++++++++++++++++ .../mtb/block-use-of-nodeport-services.md | 75 +++++++++++ .../mtb/require-always-imagepullpolicy.md | 2 +- ...quire-persistentvolumeclaim-for-storage.md | 124 ++++++++++++++++++ .../mtb/require-reclaim-policy-of-delete.md | 87 ++++++++++++ .../mtb/require-run-as-non-root-user.md | 2 +- docs/operator/mtb/sig-multitenancy-bench.md | 20 +-- 20 files changed, 880 insertions(+), 20 deletions(-) create mode 100644 docs/operator/mtb/allow-self-service-management-of-network-policies.md create mode 100644 docs/operator/mtb/allow-self-service-management-of-rolebindings.md create mode 100644 docs/operator/mtb/allow-self-service-management-of-roles.md create mode 100644 docs/operator/mtb/block-network-access-across-tenant-namespaces.md create mode 100644 docs/operator/mtb/block-use-of-existing-persistent-volumes.md create mode 100644 docs/operator/mtb/block-use-of-host-ipc.md create mode 100644 docs/operator/mtb/block-use-of-host-pid.md create mode 100644 docs/operator/mtb/block-use-of-nodeport-services.md create mode 100644 docs/operator/mtb/require-persistentvolumeclaim-for-storage.md create mode 100644 docs/operator/mtb/require-reclaim-policy-of-delete.md diff --git a/docs/operator/mtb/allow-self-service-management-of-network-policies.md b/docs/operator/mtb/allow-self-service-management-of-network-policies.md new file mode 100644 index 00000000..50ff37ad --- /dev/null +++ b/docs/operator/mtb/allow-self-service-management-of-network-policies.md @@ -0,0 +1,77 @@ +# Allow self-service management of Network Policies + +**Profile Applicability:** L2 + +**Type:** Behavioral + +**Category:** Self-Service Operations + +**Description:** Tenants should be able to perform self-service operations by creating own network policies in their namespaces. + +**Rationale:** Enables self-service management of network-policies. + +**Audit:** + +As cluster admin, create a tenant + +```yaml +kubectl create -f - < 7m5s +``` + +As tenant owner check for permissions to manage networkpolicy for each verb + +```bash +kubectl --kubeconfig alice auth can-i get networkpolicies +kubectl --kubeconfig alice auth can-i create networkpolicies +kubectl --kubeconfig alice auth can-i update networkpolicies +kubectl --kubeconfig alice auth can-i patch networkpolicies +kubectl --kubeconfig alice auth can-i delete networkpolicies +kubectl --kubeconfig alice auth can-i deletecollection networkpolicies +``` + +Each command must return 'yes' + +**Cleanup:** +As cluster admin, delete all the created resources + +```bash +kubectl --kubeconfig cluster-admin delete tenant oil +``` \ No newline at end of file diff --git a/docs/operator/mtb/allow-self-service-management-of-rolebindings.md b/docs/operator/mtb/allow-self-service-management-of-rolebindings.md new file mode 100644 index 00000000..baa4828d --- /dev/null +++ b/docs/operator/mtb/allow-self-service-management-of-rolebindings.md @@ -0,0 +1,58 @@ +# Allow self-service management of Role Bindings + +**Profile Applicability:** L2 + +**Type:** Behavioral + +**Category:** Self-Service Operations + +**Description:** Tenants should be able to perform self-service operations by creating own rolebindings in their namespaces. + +**Rationale:** Enables self-service management of roles. + +**Audit:** + +As cluster admin, create a tenant + +```yaml +kubectl create -f - < Note: make sure `PodSecurityPolicy` Admission Control is enabled on the APIs server: `--enable-admission-plugins=PodSecurityPolicy` + +Then create a ClusterRole using or granting the said item + +```yaml +kubectl create -f - << EOF +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: tenant:psp +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + resourceNames: ['tenant'] + verbs: ['use'] +EOF +``` + +And assign it to the tenant + +```yaml +kubectl apply -f - << EOF +apiVersion: capsule.clastix.io/v1beta1 +kind: Tenant +metadata: + name: oil + namespace: oil-production +spec: + owners: + - kind: User + name: alice + additionalRoleBindings: + - clusterRoleName: tenant:psp + subjects: + - kind: "Group" + apiGroup: "rbac.authorization.k8s.io" + name: "system:authenticated" +EOF + +./create-user.sh alice oil +``` + +As tenant owner, run the following command to create a namespace in the given tenant + +```bash +kubectl --kubeconfig alice create ns oil-production +kubectl --kubeconfig alice config set-context --current --namespace oil-production +``` + +As tenant owner, create a pod mounting the host IPC namespace. + +```yaml +kubectl --kubeconfig alice apply -f - << EOF +apiVersion: v1 +kind: Pod +metadata: + name: pod-with-host-ipc + namespace: oil-production +spec: + hostIPC: true + containers: + - name: busybox + image: busybox:latest + command: ["/bin/sleep", "3600"] +EOF +``` + +You must have the pod blocked by `PodSecurityPolicy`. + +**Cleanup:** +As cluster admin, delete all the created resources + +```bash +kubectl --kubeconfig cluster-admin delete tenant oil +kubectl --kubeconfig cluster-admin delete PodSecurityPolicy tenant +kubectl --kubeconfig cluster-admin delete ClusterRole tenant:psp +``` diff --git a/docs/operator/mtb/block-use-of-host-networking-and-ports.md b/docs/operator/mtb/block-use-of-host-networking-and-ports.md index 301dda33..82164d39 100644 --- a/docs/operator/mtb/block-use-of-host-networking-and-ports.md +++ b/docs/operator/mtb/block-use-of-host-networking-and-ports.md @@ -22,6 +22,8 @@ metadata: name: tenant spec: privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false hostNetwork: false hostPorts: [] # empty means no allowed host ports runAsUser: @@ -122,7 +124,7 @@ spec: EOF ``` -In both the cases, you should have the pod blocked by `PodSecurityPolicy`. +In both the cases above, you must have the pod blocked by `PodSecurityPolicy`. **Cleanup:** As cluster admin, delete all the created resources diff --git a/docs/operator/mtb/block-use-of-host-path-volumes.md b/docs/operator/mtb/block-use-of-host-path-volumes.md index 0622cc2d..95db6bd2 100644 --- a/docs/operator/mtb/block-use-of-host-path-volumes.md +++ b/docs/operator/mtb/block-use-of-host-path-volumes.md @@ -22,6 +22,8 @@ metadata: name: tenant spec: privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false volumes: # hostPath is not permitted - 'configMap' - 'emptyDir' @@ -115,7 +117,7 @@ spec: EOF ``` -You should have the pod blocked by `PodSecurityPolicy`. +You must have the pod blocked by `PodSecurityPolicy`. **Cleanup:** As cluster admin, delete all the created resources diff --git a/docs/operator/mtb/block-use-of-host-pid.md b/docs/operator/mtb/block-use-of-host-pid.md new file mode 100644 index 00000000..b7ef20c2 --- /dev/null +++ b/docs/operator/mtb/block-use-of-host-pid.md @@ -0,0 +1,115 @@ +# Block use of host PID + +**Profile Applicability:** L1 + +**Type:** Behavioral Check + +**Category:** Host Isolation + +**Description:** Tenants should not be allowed to share the host process ID (PID) namespace. + +**Rationale:** The `hostPID` setting allows pods to share the host process ID namespace allowing potential privilege escalation. Tenant pods should not be allowed to share the host PID namespace. + +**Audit:** + +As cluster admin, define a `PodSecurityPolicy` that restricts `hostPID` usage and map the policy to a tenant: + +```yaml +kubectl create -f - << EOF +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: tenant +spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + hostPID: false + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + fsGroup: + rule: RunAsAny +EOF +``` + +> Note: make sure `PodSecurityPolicy` Admission Control is enabled on the APIs server: `--enable-admission-plugins=PodSecurityPolicy` + +Then create a ClusterRole using or granting the said item + +```yaml +kubectl create -f - << EOF +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: tenant:psp +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + resourceNames: ['tenant'] + verbs: ['use'] +EOF +``` + +And assign it to the tenant + +```yaml +kubectl apply -f - << EOF +apiVersion: capsule.clastix.io/v1beta1 +kind: Tenant +metadata: + name: oil + namespace: oil-production +spec: + owners: + - kind: User + name: alice + additionalRoleBindings: + - clusterRoleName: tenant:psp + subjects: + - kind: "Group" + apiGroup: "rbac.authorization.k8s.io" + name: "system:authenticated" +EOF + +./create-user.sh alice oil +``` + +As tenant owner, run the following command to create a namespace in the given tenant + +```bash +kubectl --kubeconfig alice create ns oil-production +kubectl --kubeconfig alice config set-context --current --namespace oil-production +``` + +As tenant owner, create a pod mounting the host PID namespace. + +```yaml +kubectl --kubeconfig alice apply -f - << EOF +apiVersion: v1 +kind: Pod +metadata: + name: pod-with-host-pid + namespace: oil-production +spec: + hostPID: true + containers: + - name: busybox + image: busybox:latest + command: ["/bin/sleep", "3600"] +EOF +``` + +You must have the pod blocked by `PodSecurityPolicy`. + +**Cleanup:** +As cluster admin, delete all the created resources + +```bash +kubectl --kubeconfig cluster-admin delete tenant oil +kubectl --kubeconfig cluster-admin delete PodSecurityPolicy tenant +kubectl --kubeconfig cluster-admin delete ClusterRole tenant:psp +``` diff --git a/docs/operator/mtb/block-use-of-nodeport-services.md b/docs/operator/mtb/block-use-of-nodeport-services.md new file mode 100644 index 00000000..7d27434d --- /dev/null +++ b/docs/operator/mtb/block-use-of-nodeport-services.md @@ -0,0 +1,75 @@ +# Block use of NodePort services + +**Profile Applicability:** L1 + +**Type:** Behavioral Check + +**Category:** Host Isolation + +**Description:** Tenants should not be able to create services of type NodePort. + +**Rationale:** the service type `NodePorts` configures host ports that cannot be secured using Kubernetes network policies and require upstream firewalls. Also, multiple tenants cannot use the same host port numbers. + +**Audit:** + +As cluster admin, create a tenant + +```yaml +kubectl create -f - << EOF +apiVersion: capsule.clastix.io/v1beta1 +kind: Tenant +metadata: + name: oil +spec: + enableNodePorts: false + owners: + - kind: User + name: alice +EOF + +./create-user.sh alice oil +``` + +As tenant owner, run the following command to create a namespace in the given tenant + +```bash +kubectl --kubeconfig alice create ns oil-production +kubectl --kubeconfig alice config set-context --current --namespace oil-production +``` + +As tenant owner, creates a service in the tenant namespace having service type of `NodePort` + +```yaml +kubectl --kubeconfig alice apply -f - << EOF +apiVersion: v1 +kind: Service +metadata: + name: nginx + labels: + namespace: oil-production +spec: + ports: + - protocol: TCP + port: 8080 + targetPort: 80 + selector: + run: nginx + type: NodePort +EOF +``` + +You must receive an error message denying the request: + +``` +Error from server +Error from server (NodePort service types are forbidden for the tenant: +error when creating "STDIN": admission webhook "services.capsule.clastix.io" denied the request: +NodePort service types are forbidden for the tenant: please, reach out to the system administrators +``` + +**Cleanup:** +As cluster admin, delete all the created resources + +```bash +kubectl --kubeconfig cluster-admin delete tenant oil +``` \ No newline at end of file diff --git a/docs/operator/mtb/require-always-imagepullpolicy.md b/docs/operator/mtb/require-always-imagepullpolicy.md index f8c4a321..caa45756 100644 --- a/docs/operator/mtb/require-always-imagepullpolicy.md +++ b/docs/operator/mtb/require-always-imagepullpolicy.md @@ -55,7 +55,7 @@ spec: EOF ``` -You should receive an error message denying the request: +You must receive an error message denying the request: ``` Error from server diff --git a/docs/operator/mtb/require-persistentvolumeclaim-for-storage.md b/docs/operator/mtb/require-persistentvolumeclaim-for-storage.md new file mode 100644 index 00000000..97a629f1 --- /dev/null +++ b/docs/operator/mtb/require-persistentvolumeclaim-for-storage.md @@ -0,0 +1,124 @@ +# Require PersistentVolumeClaim for storage + +**Profile Applicability:** L1 + +**Type:** Behavioral Check + +**Category:** na + +**Description:** Tenants should not be able to use all volume types except `PersistentVolumeClaims`. + +**Rationale:** In some scenarios, it would be required to disallow usage of any core volume types except PVCs. + +**Audit:** + +As cluster admin, define a `PodSecurityPolicy` allowing only `PersistentVolumeClaim` volumes and map the policy to a tenant: + +```yaml +kubectl create -f - << EOF +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: tenant +spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + volumes: + - 'persistentVolumeClaim' + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + fsGroup: + rule: RunAsAny +EOF +``` + +> Note: make sure `PodSecurityPolicy` Admission Control is enabled on the APIs server: `--enable-admission-plugins=PodSecurityPolicy` + +Then create a ClusterRole using or granting the said item + +```yaml +kubectl create -f - << EOF +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: tenant:psp +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + resourceNames: ['tenant'] + verbs: ['use'] +EOF +``` + +And assign it to the tenant + +```yaml +kubectl apply -f - << EOF +apiVersion: capsule.clastix.io/v1beta1 +kind: Tenant +metadata: + name: oil + namespace: oil-production +spec: + owners: + - kind: User + name: alice + additionalRoleBindings: + - clusterRoleName: tenant:psp + subjects: + - kind: "Group" + apiGroup: "rbac.authorization.k8s.io" + name: "system:authenticated" +EOF + +./create-user.sh alice oil +``` + +As tenant owner, run the following command to create a namespace in the given tenant + +```bash +kubectl --kubeconfig alice create ns oil-production +kubectl --kubeconfig alice config set-context --current --namespace oil-production +``` + +As tenant owner, create a pod defining a volume of any of the core type except `PersistentVolumeClaim`. For example: + +```yaml +kubectl --kubeconfig alice apply -f - << EOF +apiVersion: v1 +kind: Pod +metadata: + name: pod-with-hostpath-volume + namespace: oil-production +spec: + containers: + - name: busybox + image: busybox:latest + command: ["/bin/sleep", "3600"] + volumeMounts: + - mountPath: /tmp + name: volume + volumes: + - name: volume + hostPath: + # directory location on host + path: /data + type: Directory +EOF +``` + +You must have the pod blocked by `PodSecurityPolicy`. + +**Cleanup:** +As cluster admin, delete all the created resources + +```bash +kubectl --kubeconfig cluster-admin delete tenant oil +kubectl --kubeconfig cluster-admin delete PodSecurityPolicy tenant +kubectl --kubeconfig cluster-admin delete ClusterRole tenant:psp +``` diff --git a/docs/operator/mtb/require-reclaim-policy-of-delete.md b/docs/operator/mtb/require-reclaim-policy-of-delete.md new file mode 100644 index 00000000..284d2725 --- /dev/null +++ b/docs/operator/mtb/require-reclaim-policy-of-delete.md @@ -0,0 +1,87 @@ +# Require PV reclaim policy of delete + +**Profile Applicability:** L1 + +**Type:** Configuration Check + +**Category:** Data Isolation + +**Description:** Force a tenant to use a Storage Class with `reclaimPolicy=Delete`. + +**Rationale:** Tenants have to be assured that their Persisten Volumes cannot be reclaimed by other tenants. + +**Audit:** + +As cluster admin, create a Storage Class with `reclaimPolicy=Delete` + +```yaml +kubectl create -f - << EOF +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: delete-policy +reclaimPolicy: Delete +provisioner: clastix.io/nfs +EOF +``` + +As cluster admin, create a tenant and assign the above Storage Class + +```yaml +kubectl create -f - << EOF +apiVersion: capsule.clastix.io/v1beta1 +kind: Tenant +metadata: + name: oil +spec: + owners: + - kind: User + name: alice + storageClasses: + allowed: + - delete-policy +EOF + +./create-user.sh alice oil +``` + +As tenant owner, run the following command to create a namespace in the given tenant + +```bash +kubectl --kubeconfig alice create ns oil-production +kubectl --kubeconfig alice config set-context --current --namespace oil-production +``` + +As tenant owner, creates a Persistent Volum Claim in the tenant namespace missing the Storage Class or using any other Storage Class: + +```yaml +kubectl --kubeconfig alice apply -f - << EOF +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: pvc + namespace: oil-production +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 12Gi +EOF +``` + +You must receive an error message denying the request: + +``` +Error from server (A valid Storage Class must be used, one of the following (delete-policy)): +error when creating "STDIN": admission webhook "pvc.capsule.clastix.io" denied the request: +A valid Storage Class must be used, one of the following (delete-policy) +``` + +**Cleanup:** +As cluster admin, delete all the created resources + +```bash +kubectl --kubeconfig cluster-admin delete tenant oil +kubectl --kubeconfig cluster-admin delete storageclass delete-policy +``` \ No newline at end of file diff --git a/docs/operator/mtb/require-run-as-non-root-user.md b/docs/operator/mtb/require-run-as-non-root-user.md index 4d768ea5..9f25d83e 100644 --- a/docs/operator/mtb/require-run-as-non-root-user.md +++ b/docs/operator/mtb/require-run-as-non-root-user.md @@ -107,7 +107,7 @@ spec: EOF ``` -You should have the pod blocked by `PodSecurityPolicy`. +You must have the pod blocked by `PodSecurityPolicy`. **Cleanup:** As cluster admin, delete all the created resources diff --git a/docs/operator/mtb/sig-multitenancy-bench.md b/docs/operator/mtb/sig-multitenancy-bench.md index 44ddc09d..d2cceb24 100644 --- a/docs/operator/mtb/sig-multitenancy-bench.md +++ b/docs/operator/mtb/sig-multitenancy-bench.md @@ -18,13 +18,13 @@ Actually, there's no yet a real standard for the multi-tenancy model in Kubernet |[Configure namespace object limits](configure-namespace-object-limits.md)|L1|v0.1.0|✓|---| |[Block use of host path volumes](block-use-of-host-path-volumes.md)|L1|v0.1.0|✓|---| |[Block use of host networking and ports](block-use-of-host-networking-and-ports.md)|L1|v0.1.0|✓|---| -|Block use of host PID|L1|v0.1.0|✓|---| -|Block use of host IPC|L1|v0.1.0|✓|---| -|Block use of NodePort services|L1|v0.1.0|✓|---| -|Require PersistentVolumeClaim for storage|L1|v0.1.0|✓|MTB draft| -|Require PV reclaim policy of delete|L1|v0.1.0|✓|MTB draft| -|Block use of existing PVs|L1|v0.1.0|✓|MTB draft| -|Block network access across tenant namespaces|L1|v0.1.0|✓|MTB draft| -|Allow self-service management of Network PoliciesL2|v0.1.0|✓|---| -|Allow self-service management of RolesL2|v0.1.0|✓|MTB draft| -|Allow self-service management of Roles Bindings|L2|v0.1.0|✓|MTB draft| +|[Block use of host PID](block-use-of-host-pid.md)|L1|v0.1.0|✓|---| +|[Block use of host IPC](block-use-of-host-ipc.md)|L1|v0.1.0|✓|---| +|[Block use of NodePort services](block-use-of-nodeport-services.md)|L1|v0.1.0|✓|---| +|[Require PersistentVolumeClaim for storage](require-persistentvolumeclaim-for-storage.md)|L1|v0.1.0|✓|MTB draft| +|[Require PV reclaim policy of delete](require-reclaim-policy-of-delete.md)|L1|v0.1.0|✓|MTB draft| +|[Block use of existing PVs](block-use-of-existing-persistent-volumes.md)|L1|v0.1.0|✓|MTB draft| +|[Block network access across tenant namespaces](block-network-access-across-tenant-namespaces.md)|L1|v0.1.0|✓|MTB draft| +|[Allow self-service management of Network Policies](allow-self-service-management-of-network-policies.md)|L2|v0.1.0|✓|---| +|[Allow self-service management of Roles](allow-self-service-management-of-roles.md)|L2|v0.1.0|✓|MTB draft| +|[Allow self-service management of Role Bindings](allow-self-service-management-of-rolebindings.md)|L2|v0.1.0|✓|MTB draft|