From 345bb8802b3ceadc3d1099b24df451418d518b39 Mon Sep 17 00:00:00 2001 From: Jerad C Date: Thu, 26 Jan 2023 11:26:05 -0600 Subject: [PATCH] remove PodSecurityPolicy when k8s version >=1.25 (#758) * add k8s 1.24 and 1.25 to test suite * remove PodSecurityPolicy when k8s version >=1.25 * add k8s 1.24 and 1.25 to automated test workflow * update docs --- .github/workflows/build-and-test.yaml | 2 +- README.md | 8 +++++- .../aws-node-termination-handler/README.md | 2 +- .../templates/psp.yaml | 2 +- .../aws-node-termination-handler/values.yaml | 2 +- config/helm/localstack/templates/psp.yaml | 2 +- config/helm/squid/templates/psp.yaml | 2 +- .../webhook-test-proxy/templates/psp.yaml | 2 +- test/k8s-local-cluster-test/provision-cluster | 27 +++++++++++++++---- test/k8s-local-cluster-test/run-test | 2 +- 10 files changed, 37 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 9190d71f..abb5647e 100755 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -168,7 +168,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - k8sVersion: ["1.18", "1.19", "1.20", "1.21", "1.22", "1.23"] + k8sVersion: ["1.18", "1.19", "1.20", "1.21", "1.22", "1.23", "1.24", "1.25"] steps: - name: Set up Go 1.x uses: actions/setup-go@v2 diff --git a/README.md b/README.md index 700de948..b4fe3c37 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,6 @@ The `enableSqsTerminationDraining` must be set to false for these configuration The Queue Processor Mode does not allow for fine-grained configuration of which events are handled through helm configuration keys. Instead, you can modify your Amazon EventBridge rules to not send certain types of events to the SQS Queue so that NTH does not process those events. All events when operating in Queue Processor mode are Cordoned and Drained unless the `cordon-only` flag is set to true. - The `enableSqsTerminationDraining` flag turns on Queue Processor Mode. When Queue Processor Mode is enabled, IMDS mode will be disabled, even if you explicitly enabled any of the IMDS configuration keys. NTH cannot respond to queue events AND monitor IMDS paths. In this case, it is safe to disable IMDS for the NTH pod.
@@ -105,6 +104,9 @@ The `enableSqsTerminationDraining` flag turns on Queue Processor Mode. When Queu The termination handler DaemonSet installs into your cluster a [ServiceAccount](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/), [ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/), [ClusterRoleBinding](https://kubernetes.io/docs/reference/access-authn-authz/rbac/), and a [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/). All four of these Kubernetes constructs are required for the termination handler to run properly. +#### Pod Security Admission + +When using Kubernetes [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) it is recommended to assign the `[privileged](https://kubernetes.io/docs/concepts/security/pod-security-standards/#privileged)` level. #### Kubectl Apply @@ -376,6 +378,10 @@ IAM Policy for aws-node-termination-handler Deployment: ### Installation +#### Pod Security Admission + +When using Kubernetes [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) it is recommended to assign the `[baseline](https://kubernetes.io/docs/concepts/security/pod-security-standards/#baseline)` level. + #### Helm The easiest and most commonly used method to configure the termination handler is via [helm](https://helm.sh/). The chart for this project is hosted in the [eks-charts](https://github.com/aws/eks-charts) repository. diff --git a/config/helm/aws-node-termination-handler/README.md b/config/helm/aws-node-termination-handler/README.md index 3e2e74e0..2fe7d391 100644 --- a/config/helm/aws-node-termination-handler/README.md +++ b/config/helm/aws-node-termination-handler/README.md @@ -56,7 +56,7 @@ The configuration in this table applies to all AWS Node Termination Handler mode | `serviceAccount.name` | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the full name template. | `nil` | | `serviceAccount.annotations` | Annotations to add to the service account. | `{}` | | `rbac.create` | If `true`, create the RBAC resources. | `true` | -| `rbac.pspEnabled` | If `true`, create a pod security policy resource. | `true` | +| `rbac.pspEnabled` | If `true`, create a pod security policy resource. Note: `PodSecurityPolicy`s will not be created when Kubernetes version is 1.25 or later. | `true` | | `customLabels` | Labels to add to all resource metadata. | `{}` | | `podLabels` | Labels to add to the pod. | `{}` | | `podAnnotations` | Annotations to add to the pod. | `{}` | diff --git a/config/helm/aws-node-termination-handler/templates/psp.yaml b/config/helm/aws-node-termination-handler/templates/psp.yaml index 70c576e8..c84d69f3 100644 --- a/config/helm/aws-node-termination-handler/templates/psp.yaml +++ b/config/helm/aws-node-termination-handler/templates/psp.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.pspEnabled }} +{{- if and (.Values.rbac.pspEnabled) (semverCompare "<1.25-0" .Capabilities.KubeVersion.GitVersion) }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: diff --git a/config/helm/aws-node-termination-handler/values.yaml b/config/helm/aws-node-termination-handler/values.yaml index 4ec9e1dd..ea7b7f77 100644 --- a/config/helm/aws-node-termination-handler/values.yaml +++ b/config/helm/aws-node-termination-handler/values.yaml @@ -23,7 +23,7 @@ serviceAccount: rbac: # Specifies whether RBAC resources should be created create: true - # Specifies if PodSecurityPolicy resources should be created + # Specifies if PodSecurityPolicy resources should be created. PodSecurityPolicy will not be created when Kubernetes version is 1.25 or later. pspEnabled: true customLabels: {} diff --git a/config/helm/localstack/templates/psp.yaml b/config/helm/localstack/templates/psp.yaml index d50afed7..a3a3c92b 100644 --- a/config/helm/localstack/templates/psp.yaml +++ b/config/helm/localstack/templates/psp.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.pspEnabled }} +{{- if and (.Values.rbac.pspEnabled) (semverCompare "<1.25-0" .Capabilities.KubeVersion.GitVersion) }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: diff --git a/config/helm/squid/templates/psp.yaml b/config/helm/squid/templates/psp.yaml index abf07ecd..00157a97 100644 --- a/config/helm/squid/templates/psp.yaml +++ b/config/helm/squid/templates/psp.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.pspEnabled }} +{{- if and (.Values.rbac.pspEnabled) (semverCompare "<1.25-0" .Capabilities.KubeVersion.GitVersion) }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: diff --git a/config/helm/webhook-test-proxy/templates/psp.yaml b/config/helm/webhook-test-proxy/templates/psp.yaml index d511e1cf..c0df5e46 100644 --- a/config/helm/webhook-test-proxy/templates/psp.yaml +++ b/config/helm/webhook-test-proxy/templates/psp.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.pspEnabled }} +{{- if and (.Values.rbac.pspEnabled) (semverCompare "<1.25-0" .Capabilities.KubeVersion.GitVersion) }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: diff --git a/test/k8s-local-cluster-test/provision-cluster b/test/k8s-local-cluster-test/provision-cluster index 773df544..4d493d4d 100755 --- a/test/k8s-local-cluster-test/provision-cluster +++ b/test/k8s-local-cluster-test/provision-cluster @@ -7,7 +7,12 @@ TEST_ID=$(uuidgen | cut -d'-' -f1 | tr '[:upper:]' '[:lower:]') CLUSTER_NAME_BASE=$(uuidgen | cut -d'-' -f1 | tr '[:upper:]' '[:lower:]') OVERRIDE_PATH=0 KIND_CONFIG_FILE=$SCRIPTPATH/kind-three-node-cluster.yaml +use_psp=false +# shellcheck disable=SC2034 +K8_1_25="kindest/node:v1.25.3@sha256:f1de3b0670462f43280114eccceab8bf1b9576d2afe0582f8f74529da6fd0365" +# shellcheck disable=SC2034 +K8_1_24="kindest/node:v1.24.7@sha256:5c015142d9b60a0f6c45573f809957076514e38ec973565e2b2fe828b91597f5" # shellcheck disable=SC2034 K8_1_23="kindest/node:v1.23.5@sha256:1a72748086bc24ed6163de1d1e33cc0e2eb5a1eb5ebffdb15b53c3bcd5376a6f" # shellcheck disable=SC2034 @@ -23,8 +28,8 @@ K8_1_18="kindest/node:v1.18.19@sha256:7af1492e19b3192a79f606e43c35fb741e520d195f K8_VERSION="$K8_1_20" KUBECTL_VERSION=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt) -KIND_VERSION="0.11.1" -HELM_VERSION="3.7.1" +KIND_VERSION="0.17.0" +HELM_VERSION="3.10.0" echoerr() { echo "$@" 1>&2; } @@ -55,7 +60,12 @@ while getopts "b:i:v:k:o" opt; do echoerr "👉 Test Run: $TEST_ID 👈" ;; v ) # K8s version to provision - OPTARG="K8_`echo $OPTARG | sed 's/\./\_/g'`" + maj_ver=`echo $OPTARG | cut -d '.' -f 1` + min_ver=`echo $OPTARG | cut -d '.' -f 2` + if [[ $maj_ver -eq 1 && $min_ver -lt 25 ]]; then + use_psp=true + fi + OPTARG="K8_${maj_ver}_${min_ver}" if [ ! -z ${OPTARG+x} ]; then K8_VERSION=${!OPTARG} else @@ -124,12 +134,19 @@ fi # Disable spinners and color in kind output export TERM=dumb echoerr "🥑 Creating k8s cluster using \"kind\"" +if [[ "$use_psp" = false ]]; then + no_psp_kind_config_file="${TMP_DIR}/`basename $KIND_CONFIG_FILE`" + cat $KIND_CONFIG_FILE | sed 's/,PodSecurityPolicy//' > "${no_psp_kind_config_file}" + KIND_CONFIG_FILE="${no_psp_kind_config_file}" +fi retry 3 kind create cluster --name "$CLUSTER_NAME" --image $K8_VERSION --config "$KIND_CONFIG_FILE" --kubeconfig $TMP_DIR/kubeconfig 1>&2 echo "$CLUSTER_NAME" > "$TMP_DIR/clustername" echoerr "👍 Created k8s cluster using \"kind\"" -kubectl apply -f "$SCRIPTPATH/psp-default.yaml" --context "kind-$CLUSTER_NAME" --kubeconfig "$TMP_DIR/kubeconfig" 1>&2 -kubectl apply -f "$SCRIPTPATH/psp-privileged.yaml" --context "kind-$CLUSTER_NAME" --kubeconfig "$TMP_DIR/kubeconfig" 1>&2 +if [[ "$use_psp" = true ]]; then + kubectl apply -f "$SCRIPTPATH/psp-default.yaml" --context "kind-$CLUSTER_NAME" --kubeconfig "$TMP_DIR/kubeconfig" 1>&2 + kubectl apply -f "$SCRIPTPATH/psp-privileged.yaml" --context "kind-$CLUSTER_NAME" --kubeconfig "$TMP_DIR/kubeconfig" 1>&2 +fi echo "$TMP_DIR" diff --git a/test/k8s-local-cluster-test/run-test b/test/k8s-local-cluster-test/run-test index f58c2140..7b294658 100755 --- a/test/k8s-local-cluster-test/run-test +++ b/test/k8s-local-cluster-test/run-test @@ -136,7 +136,7 @@ USAGE=$(cat << 'EOM' -n Node Termination Handler Docker Image -d use GOPROXY=direct to bypass proxy.golang.org -o Override path w/ your own kubectl and kind binaries - -v Kubernetes Version (Default: 1.20) [1.18, 1.19, 1.20, 1.21, 1.22, and 1.23] + -v Kubernetes Version (Default: 1.20) [1.18, 1.19, 1.20, 1.21, 1.22, 1.23, 1.24, and 1.25] -w Webhook Docker Image EOM