From 0d0572308cf23c388f1d72bd25af61e240f24717 Mon Sep 17 00:00:00 2001 From: Petro Protsakh Date: Wed, 23 Oct 2024 15:49:47 +0300 Subject: [PATCH 1/2] SCALRCORE-32580 Add option to restrict access to AWS IMDS via NetworkPolicy --- charts/agent-k8s/README.md | 1 + charts/agent-k8s/templates/networkpolicy.yaml | 21 +++++++++++++++++++ charts/agent-k8s/values.yaml | 2 ++ 3 files changed, 24 insertions(+) create mode 100644 charts/agent-k8s/templates/networkpolicy.yaml diff --git a/charts/agent-k8s/README.md b/charts/agent-k8s/README.md index 5a1ed60..bb89ae2 100644 --- a/charts/agent-k8s/README.md +++ b/charts/agent-k8s/README.md @@ -134,6 +134,7 @@ as its data home. | agent.kubernetes_task_annotations | object | `{}` | Extra annotations to apply to the agent task pods. | | agent.kubernetes_task_labels | object | `{}` | Extra labels to apply to the agent task pods. | | agent.log_format | string | `"json"` | The log formatter. Options: "plain" or "dev" or "json". | +| agent.restrictAwsMetadata | bool | `false` | Apply NetworkPolicy to agent pod that denies access to AWS IMDS | | agent.token | string | `""` | The agent pool token. | | agent.tokenExistingSecret | string | `""` | The name of the secret containing the agent pool token. Secret is created if left empty. | | agent.tokenExistingSecretKey | string | `"token"` | The key of the secret containing the agent pool token. | diff --git a/charts/agent-k8s/templates/networkpolicy.yaml b/charts/agent-k8s/templates/networkpolicy.yaml new file mode 100644 index 0000000..127219b --- /dev/null +++ b/charts/agent-k8s/templates/networkpolicy.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: agent-k8s-network-policy + namespace: {{ .Release.Namespace | quote }} +spec: + podSelector: + matchLabels: + {{- include "agent-k8s.selectorLabels" . | nindent 6 }} + policyTypes: + - Egress + egress: + - to: + - ipBlock: + # Allow all egress traffic by default + cidr: 0.0.0.0/0 + except: + {{- if .Values.agent.restrictAwsMetadata }} + # Deny access to AWS IMDS + - 169.254.169.254/32 + {{- end }} diff --git a/charts/agent-k8s/values.yaml b/charts/agent-k8s/values.yaml index 97f2765..a7b4013 100644 --- a/charts/agent-k8s/values.yaml +++ b/charts/agent-k8s/values.yaml @@ -67,6 +67,8 @@ agent: kubernetes_task_annotations: {} # -- Enable automatic mounting of the service account token into the agent task pods. automount_service_account_token: false + # -- Apply NetworkPolicy to agent pod that denies access to AWS IMDS + restrictAwsMetadata: false imagePullSecrets: [] nameOverride: "" From 812706f8422cd755c07804c6facba8704a164d9d Mon Sep 17 00:00:00 2001 From: Petro Protsakh Date: Mon, 28 Oct 2024 14:11:32 +0200 Subject: [PATCH 2/2] SCALRCORE-32580 Update policy, add readme entry --- charts/agent-k8s/CHANGELOG.md | 4 ++++ charts/agent-k8s/README.md | 21 ++++++++++++++++++- charts/agent-k8s/README.md.gotmpl | 19 +++++++++++++++++ charts/agent-k8s/templates/networkpolicy.yaml | 6 +++--- charts/agent-k8s/values.yaml | 5 +++-- 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/charts/agent-k8s/CHANGELOG.md b/charts/agent-k8s/CHANGELOG.md index cc6b310..3ba760d 100644 --- a/charts/agent-k8s/CHANGELOG.md +++ b/charts/agent-k8s/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Added + +- Added `restrictMetadataService` option. When set to true, applies pod network policy that blocks outbound access to instance metadata service. + ## [v0.5.19] ### Updated diff --git a/charts/agent-k8s/README.md b/charts/agent-k8s/README.md index bb89ae2..261f3dc 100644 --- a/charts/agent-k8s/README.md +++ b/charts/agent-k8s/README.md @@ -107,6 +107,25 @@ The EFS storage will be mounted in all worker containers at the `agent.data_home for Runs will inherit the EFS configuration. The controller will continue to use an ephemeral directory as its data home. +## Restrict Access to VM Metadata Service + +The chart includes an optional feature to restrict the pods from accessing the VM metadata service at 169.254.169.254, which is common for both AWS and GCP environments. + +To enable it, use the `restrictMetadataService` option: + +```console +$ helm upgrade ... \ + --set restrictMetadataService=true +``` + +With this option enabled, a Kubernetes NetworkPolicy is applied to the agent pods that denies egress traffic to 169.254.169.254/32, blocking access to the VM metadata service. All other outbound traffic is allowed. + +### Limitations + +Ensure that your cluster is using a CNI plugin that supports egress NetworkPolicies. Example: Calico, Cilium, or native GKE NetworkPolicy provider for supported versions. + +If your cluster doesn't currently support egress NetworkPolicies, you may need to recreate it with the appropriate settings. + ## Maintainers | Name | Email | Url | @@ -134,7 +153,6 @@ as its data home. | agent.kubernetes_task_annotations | object | `{}` | Extra annotations to apply to the agent task pods. | | agent.kubernetes_task_labels | object | `{}` | Extra labels to apply to the agent task pods. | | agent.log_format | string | `"json"` | The log formatter. Options: "plain" or "dev" or "json". | -| agent.restrictAwsMetadata | bool | `false` | Apply NetworkPolicy to agent pod that denies access to AWS IMDS | | agent.token | string | `""` | The agent pool token. | | agent.tokenExistingSecret | string | `""` | The name of the secret containing the agent pool token. Secret is created if left empty. | | agent.tokenExistingSecretKey | string | `"token"` | The key of the secret containing the agent pool token. | @@ -156,6 +174,7 @@ as its data home. | resources.limits.memory | string | `"1024Mi"` | | | resources.requests.cpu | string | `"250m"` | | | resources.requests.memory | string | `"256Mi"` | | +| restrictMetadataService | bool | `false` | Apply NetworkPolicy to an agent pod that denies access to VM metadata service address (169.254.169.254) | | securityContext | object | `{"runAsGroup":0,"runAsUser":0}` | The Agent Pods security context. | | serviceAccount.annotations | object | `{}` | Annotations to add to the service account | | serviceAccount.create | bool | `true` | Specifies whether a service account should be created | diff --git a/charts/agent-k8s/README.md.gotmpl b/charts/agent-k8s/README.md.gotmpl index 7e883fd..200b65a 100644 --- a/charts/agent-k8s/README.md.gotmpl +++ b/charts/agent-k8s/README.md.gotmpl @@ -99,6 +99,25 @@ The EFS storage will be mounted in all worker containers at the `agent.data_home for Runs will inherit the EFS configuration. The controller will continue to use an ephemeral directory as its data home. +## Restrict Access to VM Metadata Service + +The chart includes an optional feature to restrict the pods from accessing the VM metadata service at 169.254.169.254, that is common for both AWS and GCP environments. + +To enable it, use the `restrictMetadataService` option: + +```console +$ helm upgrade ... \ + --set restrictMetadataService=true +``` + +With this option enabled, a Kubernetes NetworkPolicy is applied to the agent pods that denies egress traffic to 169.254.169.254/32, blocking access to the VM metadata service. All other outbound traffic is allowed. + +### Limitations + +Ensure that your cluster is using a CNI plugin that supports egress NetworkPolicies. Example: Calico, Cilium, or native GKE NetworkPolicy provider for supported versions. + +If your cluster doesn't currently support egress NetworkPolicies, you may need to recreate it with the appropriate settings. + {{ template "chart.maintainersSection" . }} {{ template "chart.requirementsSection" . }} diff --git a/charts/agent-k8s/templates/networkpolicy.yaml b/charts/agent-k8s/templates/networkpolicy.yaml index 127219b..c632465 100644 --- a/charts/agent-k8s/templates/networkpolicy.yaml +++ b/charts/agent-k8s/templates/networkpolicy.yaml @@ -1,3 +1,4 @@ +{{- if .Values.restrictMetadataService }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -15,7 +16,6 @@ spec: # Allow all egress traffic by default cidr: 0.0.0.0/0 except: - {{- if .Values.agent.restrictAwsMetadata }} - # Deny access to AWS IMDS + # Deny access to IMDS - 169.254.169.254/32 - {{- end }} +{{- end }} diff --git a/charts/agent-k8s/values.yaml b/charts/agent-k8s/values.yaml index a7b4013..c11bc41 100644 --- a/charts/agent-k8s/values.yaml +++ b/charts/agent-k8s/values.yaml @@ -67,8 +67,6 @@ agent: kubernetes_task_annotations: {} # -- Enable automatic mounting of the service account token into the agent task pods. automount_service_account_token: false - # -- Apply NetworkPolicy to agent pod that denies access to AWS IMDS - restrictAwsMetadata: false imagePullSecrets: [] nameOverride: "" @@ -112,6 +110,9 @@ securityContext: # -- The Agent Pods annotations. podAnnotations: {} +# -- Apply NetworkPolicy to an agent pod that denies access to VM metadata service address (169.254.169.254) +restrictMetadataService: false + # -- Provides the amount of grace time prior to the agent-k8s container being forcibly terminated when marked for deletion or restarted. terminationGracePeriodSeconds: 3660