From e1190e2754dc6d695f3995c1eeab56baa1da7faa Mon Sep 17 00:00:00 2001 From: Jerad C Date: Tue, 24 Jan 2023 14:26:25 -0600 Subject: [PATCH 1/4] add k8s 1.24 and 1.25 to test suite --- test/k8s-local-cluster-test/provision-cluster | 8 ++++++-- test/k8s-local-cluster-test/run-test | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/test/k8s-local-cluster-test/provision-cluster b/test/k8s-local-cluster-test/provision-cluster index 773df544..f6422ac3 100755 --- a/test/k8s-local-cluster-test/provision-cluster +++ b/test/k8s-local-cluster-test/provision-cluster @@ -8,6 +8,10 @@ CLUSTER_NAME_BASE=$(uuidgen | cut -d'-' -f1 | tr '[:upper:]' '[:lower:]') OVERRIDE_PATH=0 KIND_CONFIG_FILE=$SCRIPTPATH/kind-three-node-cluster.yaml +# shellcheck disable=SC2034 +K8_1_25="kindest/node:v1.25.3@sha256:f1de3b0670462f43280114eccceab8bf1b9576d2afe0582f8f74529da6fd0365" +# shellcheck disable=SC2034 +K8_1_24="kindest/node:v1.24.7@sha256:5c015142d9b60a0f6c45573f809957076514e38ec973565e2b2fe828b91597f5" # shellcheck disable=SC2034 K8_1_23="kindest/node:v1.23.5@sha256:1a72748086bc24ed6163de1d1e33cc0e2eb5a1eb5ebffdb15b53c3bcd5376a6f" # shellcheck disable=SC2034 @@ -23,8 +27,8 @@ K8_1_18="kindest/node:v1.18.19@sha256:7af1492e19b3192a79f606e43c35fb741e520d195f K8_VERSION="$K8_1_20" KUBECTL_VERSION=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt) -KIND_VERSION="0.11.1" -HELM_VERSION="3.7.1" +KIND_VERSION="0.17.0" +HELM_VERSION="3.10.0" echoerr() { echo "$@" 1>&2; } diff --git a/test/k8s-local-cluster-test/run-test b/test/k8s-local-cluster-test/run-test index f58c2140..7b294658 100755 --- a/test/k8s-local-cluster-test/run-test +++ b/test/k8s-local-cluster-test/run-test @@ -136,7 +136,7 @@ USAGE=$(cat << 'EOM' -n Node Termination Handler Docker Image -d use GOPROXY=direct to bypass proxy.golang.org -o Override path w/ your own kubectl and kind binaries - -v Kubernetes Version (Default: 1.20) [1.18, 1.19, 1.20, 1.21, 1.22, and 1.23] + -v Kubernetes Version (Default: 1.20) [1.18, 1.19, 1.20, 1.21, 1.22, 1.23, 1.24, and 1.25] -w Webhook Docker Image EOM From 0eb5b0d1896a1a11a9fadcfc01db3ee70db2d9cd Mon Sep 17 00:00:00 2001 From: Jerad C Date: Tue, 24 Jan 2023 14:27:55 -0600 Subject: [PATCH 2/4] disable PodSecurityPolicy when k8s version >=1.24 --- .../templates/psp.yaml | 2 +- config/helm/localstack/templates/psp.yaml | 2 +- config/helm/squid/templates/psp.yaml | 2 +- .../webhook-test-proxy/templates/psp.yaml | 2 +- test/k8s-local-cluster-test/provision-cluster | 19 ++++++++++++++++--- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/config/helm/aws-node-termination-handler/templates/psp.yaml b/config/helm/aws-node-termination-handler/templates/psp.yaml index 70c576e8..c84d69f3 100644 --- a/config/helm/aws-node-termination-handler/templates/psp.yaml +++ b/config/helm/aws-node-termination-handler/templates/psp.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.pspEnabled }} +{{- if and (.Values.rbac.pspEnabled) (semverCompare "<1.25-0" .Capabilities.KubeVersion.GitVersion) }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: diff --git a/config/helm/localstack/templates/psp.yaml b/config/helm/localstack/templates/psp.yaml index d50afed7..a3a3c92b 100644 --- a/config/helm/localstack/templates/psp.yaml +++ b/config/helm/localstack/templates/psp.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.pspEnabled }} +{{- if and (.Values.rbac.pspEnabled) (semverCompare "<1.25-0" .Capabilities.KubeVersion.GitVersion) }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: diff --git a/config/helm/squid/templates/psp.yaml b/config/helm/squid/templates/psp.yaml index abf07ecd..00157a97 100644 --- a/config/helm/squid/templates/psp.yaml +++ b/config/helm/squid/templates/psp.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.pspEnabled }} +{{- if and (.Values.rbac.pspEnabled) (semverCompare "<1.25-0" .Capabilities.KubeVersion.GitVersion) }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: diff --git a/config/helm/webhook-test-proxy/templates/psp.yaml b/config/helm/webhook-test-proxy/templates/psp.yaml index d511e1cf..c0df5e46 100644 --- a/config/helm/webhook-test-proxy/templates/psp.yaml +++ b/config/helm/webhook-test-proxy/templates/psp.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.pspEnabled }} +{{- if and (.Values.rbac.pspEnabled) (semverCompare "<1.25-0" .Capabilities.KubeVersion.GitVersion) }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: diff --git a/test/k8s-local-cluster-test/provision-cluster b/test/k8s-local-cluster-test/provision-cluster index f6422ac3..4d493d4d 100755 --- a/test/k8s-local-cluster-test/provision-cluster +++ b/test/k8s-local-cluster-test/provision-cluster @@ -7,6 +7,7 @@ TEST_ID=$(uuidgen | cut -d'-' -f1 | tr '[:upper:]' '[:lower:]') CLUSTER_NAME_BASE=$(uuidgen | cut -d'-' -f1 | tr '[:upper:]' '[:lower:]') OVERRIDE_PATH=0 KIND_CONFIG_FILE=$SCRIPTPATH/kind-three-node-cluster.yaml +use_psp=false # shellcheck disable=SC2034 K8_1_25="kindest/node:v1.25.3@sha256:f1de3b0670462f43280114eccceab8bf1b9576d2afe0582f8f74529da6fd0365" @@ -59,7 +60,12 @@ while getopts "b:i:v:k:o" opt; do echoerr "👉 Test Run: $TEST_ID 👈" ;; v ) # K8s version to provision - OPTARG="K8_`echo $OPTARG | sed 's/\./\_/g'`" + maj_ver=`echo $OPTARG | cut -d '.' -f 1` + min_ver=`echo $OPTARG | cut -d '.' -f 2` + if [[ $maj_ver -eq 1 && $min_ver -lt 25 ]]; then + use_psp=true + fi + OPTARG="K8_${maj_ver}_${min_ver}" if [ ! -z ${OPTARG+x} ]; then K8_VERSION=${!OPTARG} else @@ -128,12 +134,19 @@ fi # Disable spinners and color in kind output export TERM=dumb echoerr "🥑 Creating k8s cluster using \"kind\"" +if [[ "$use_psp" = false ]]; then + no_psp_kind_config_file="${TMP_DIR}/`basename $KIND_CONFIG_FILE`" + cat $KIND_CONFIG_FILE | sed 's/,PodSecurityPolicy//' > "${no_psp_kind_config_file}" + KIND_CONFIG_FILE="${no_psp_kind_config_file}" +fi retry 3 kind create cluster --name "$CLUSTER_NAME" --image $K8_VERSION --config "$KIND_CONFIG_FILE" --kubeconfig $TMP_DIR/kubeconfig 1>&2 echo "$CLUSTER_NAME" > "$TMP_DIR/clustername" echoerr "👍 Created k8s cluster using \"kind\"" -kubectl apply -f "$SCRIPTPATH/psp-default.yaml" --context "kind-$CLUSTER_NAME" --kubeconfig "$TMP_DIR/kubeconfig" 1>&2 -kubectl apply -f "$SCRIPTPATH/psp-privileged.yaml" --context "kind-$CLUSTER_NAME" --kubeconfig "$TMP_DIR/kubeconfig" 1>&2 +if [[ "$use_psp" = true ]]; then + kubectl apply -f "$SCRIPTPATH/psp-default.yaml" --context "kind-$CLUSTER_NAME" --kubeconfig "$TMP_DIR/kubeconfig" 1>&2 + kubectl apply -f "$SCRIPTPATH/psp-privileged.yaml" --context "kind-$CLUSTER_NAME" --kubeconfig "$TMP_DIR/kubeconfig" 1>&2 +fi echo "$TMP_DIR" From c174770a55fbec58c3c420455f2612f5525702ff Mon Sep 17 00:00:00 2001 From: Jerad C Date: Tue, 24 Jan 2023 14:31:52 -0600 Subject: [PATCH 3/4] add k8s 1.24 and 1.25 to automated test workflow --- .github/workflows/build-and-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 9190d71f..abb5647e 100755 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -168,7 +168,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - k8sVersion: ["1.18", "1.19", "1.20", "1.21", "1.22", "1.23"] + k8sVersion: ["1.18", "1.19", "1.20", "1.21", "1.22", "1.23", "1.24", "1.25"] steps: - name: Set up Go 1.x uses: actions/setup-go@v2 From ddfe77f5e4969d47b2f8290c0d5bde25c755ce8a Mon Sep 17 00:00:00 2001 From: Jerad C Date: Wed, 25 Jan 2023 14:55:32 -0600 Subject: [PATCH 4/4] update docs --- README.md | 8 +++++++- config/helm/aws-node-termination-handler/README.md | 2 +- config/helm/aws-node-termination-handler/values.yaml | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 700de948..b4fe3c37 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,6 @@ The `enableSqsTerminationDraining` must be set to false for these configuration The Queue Processor Mode does not allow for fine-grained configuration of which events are handled through helm configuration keys. Instead, you can modify your Amazon EventBridge rules to not send certain types of events to the SQS Queue so that NTH does not process those events. All events when operating in Queue Processor mode are Cordoned and Drained unless the `cordon-only` flag is set to true. - The `enableSqsTerminationDraining` flag turns on Queue Processor Mode. When Queue Processor Mode is enabled, IMDS mode will be disabled, even if you explicitly enabled any of the IMDS configuration keys. NTH cannot respond to queue events AND monitor IMDS paths. In this case, it is safe to disable IMDS for the NTH pod.
@@ -105,6 +104,9 @@ The `enableSqsTerminationDraining` flag turns on Queue Processor Mode. When Queu The termination handler DaemonSet installs into your cluster a [ServiceAccount](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/), [ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/), [ClusterRoleBinding](https://kubernetes.io/docs/reference/access-authn-authz/rbac/), and a [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/). All four of these Kubernetes constructs are required for the termination handler to run properly. +#### Pod Security Admission + +When using Kubernetes [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) it is recommended to assign the `[privileged](https://kubernetes.io/docs/concepts/security/pod-security-standards/#privileged)` level. #### Kubectl Apply @@ -376,6 +378,10 @@ IAM Policy for aws-node-termination-handler Deployment: ### Installation +#### Pod Security Admission + +When using Kubernetes [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) it is recommended to assign the `[baseline](https://kubernetes.io/docs/concepts/security/pod-security-standards/#baseline)` level. + #### Helm The easiest and most commonly used method to configure the termination handler is via [helm](https://helm.sh/). The chart for this project is hosted in the [eks-charts](https://github.com/aws/eks-charts) repository. diff --git a/config/helm/aws-node-termination-handler/README.md b/config/helm/aws-node-termination-handler/README.md index 3e2e74e0..2fe7d391 100644 --- a/config/helm/aws-node-termination-handler/README.md +++ b/config/helm/aws-node-termination-handler/README.md @@ -56,7 +56,7 @@ The configuration in this table applies to all AWS Node Termination Handler mode | `serviceAccount.name` | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the full name template. | `nil` | | `serviceAccount.annotations` | Annotations to add to the service account. | `{}` | | `rbac.create` | If `true`, create the RBAC resources. | `true` | -| `rbac.pspEnabled` | If `true`, create a pod security policy resource. | `true` | +| `rbac.pspEnabled` | If `true`, create a pod security policy resource. Note: `PodSecurityPolicy`s will not be created when Kubernetes version is 1.25 or later. | `true` | | `customLabels` | Labels to add to all resource metadata. | `{}` | | `podLabels` | Labels to add to the pod. | `{}` | | `podAnnotations` | Annotations to add to the pod. | `{}` | diff --git a/config/helm/aws-node-termination-handler/values.yaml b/config/helm/aws-node-termination-handler/values.yaml index 4ec9e1dd..ea7b7f77 100644 --- a/config/helm/aws-node-termination-handler/values.yaml +++ b/config/helm/aws-node-termination-handler/values.yaml @@ -23,7 +23,7 @@ serviceAccount: rbac: # Specifies whether RBAC resources should be created create: true - # Specifies if PodSecurityPolicy resources should be created + # Specifies if PodSecurityPolicy resources should be created. PodSecurityPolicy will not be created when Kubernetes version is 1.25 or later. pspEnabled: true customLabels: {}