diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d5e904..8052f00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Updated all templates with changes from upstream release v1.11.9 + ## [0.4.2] - 2022-10-14 ### Fixed diff --git a/helm/cilium/files/envoy/envoy.yaml b/helm/cilium/files/envoy/envoy.yaml deleted file mode 100644 index 6d25613..0000000 --- a/helm/cilium/files/envoy/envoy.yaml +++ /dev/null @@ -1,71 +0,0 @@ -static_resources: - listeners: - - name: listener_hubble_ui - address: - socket_address: - address: 0.0.0.0 - port_value: 8081 - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - codec_type: auto - stat_prefix: ingress_http - route_config: - name: local_route - virtual_hosts: - - name: local_service - domains: ["*"] - routes: - - match: - prefix: "/api/" - route: - cluster: backend - prefix_rewrite: "/" - timeout: 0s - max_stream_duration: - grpc_timeout_header_max: 0s - - match: - prefix: "/" - route: - cluster: frontend - cors: - allow_origin_string_match: - - prefix: "*" - allow_methods: GET, PUT, DELETE, POST, OPTIONS - allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout - max_age: "1728000" - expose_headers: grpc-status,grpc-message - http_filters: - - name: envoy.filters.http.grpc_web - - name: envoy.filters.http.cors - - name: envoy.filters.http.router - clusters: - - name: frontend - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: frontend - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 8080 - - name: backend - connect_timeout: 0.25s - type: logical_dns - lb_policy: round_robin - http2_protocol_options: {} - load_assignment: - cluster_name: backend - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 8090 diff --git a/helm/cilium/files/nodeinit/prestop.bash b/helm/cilium/files/nodeinit/prestop.bash index 4516604..caf4ba6 100644 --- a/helm/cilium/files/nodeinit/prestop.bash +++ b/helm/cilium/files/nodeinit/prestop.bash @@ -23,7 +23,7 @@ if ip link show cilium_host; then fi {{- if not (eq .Values.nodeinit.bootstrapFile "") }} -rm -f {{ .Values.nodeinit.bootstrapFile }} +rm -f {{ .Values.nodeinit.bootstrapFile | quote }} {{- end }} rm -f /tmp/node-init.cilium.io @@ -46,7 +46,7 @@ echo "Restarting the kubelet" systemctl restart kubelet {{- end }} -{{- if (and .Values.gke.enabled (or .Values.masquerade .Values.gke.disableDefaultSnat))}} +{{- if (and .Values.gke.enabled (or .Values.enableIPv4Masquerade .Values.gke.disableDefaultSnat))}} # If the IP-MASQ chain exists, add back default jump rule from the GKE instance configure script if iptables -w -t nat -L IP-MASQ > /dev/null; then iptables -w -t nat -A POSTROUTING -m comment --comment "ip-masq: ensure nat POSTROUTING directs all non-LOCAL destination traffic to our custom IP-MASQ chain" -m addrtype ! --dst-type LOCAL -j IP-MASQ diff --git a/helm/cilium/files/nodeinit/startup.bash b/helm/cilium/files/nodeinit/startup.bash index e87eda5..91bc5d2 100644 --- a/helm/cilium/files/nodeinit/startup.bash +++ b/helm/cilium/files/nodeinit/startup.bash @@ -22,9 +22,11 @@ fi {{- end }} {{- if .Values.nodeinit.reconfigureKubelet }} -# Check if we're running on a GKE containerd flavor. +# Check if we're running on a GKE containerd flavor as indicated by the presence +# of the '--container-runtime-endpoint' flag in '/etc/default/kubelet'. GKE_KUBERNETES_BIN_DIR="/home/kubernetes/bin" -if [[ -f "${GKE_KUBERNETES_BIN_DIR}/gke" ]] && command -v containerd &>/dev/null; then +KUBELET_DEFAULTS_FILE="/etc/default/kubelet" +if [[ -f "${GKE_KUBERNETES_BIN_DIR}/gke" ]] && [[ $(grep -cF -- '--container-runtime-endpoint' "${KUBELET_DEFAULTS_FILE}") == "1" ]]; then echo "GKE *_containerd flavor detected..." # (GKE *_containerd) Upon node restarts, GKE's containerd images seem to reset @@ -98,13 +100,13 @@ else # (Generic) Alter the kubelet configuration to run in CNI mode echo "Changing kubelet configuration to --network-plugin=cni --cni-bin-dir={{ .Values.cni.binPath }}" mkdir -p {{ .Values.cni.binPath }} - sed -i "s:--network-plugin=kubenet:--network-plugin=cni\ --cni-bin-dir={{ .Values.cni.binPath }}:g" /etc/default/kubelet + sed -i "s:--network-plugin=kubenet:--network-plugin=cni\ --cni-bin-dir={{ .Values.cni.binPath }}:g" "${KUBELET_DEFAULTS_FILE}" fi echo "Restarting the kubelet..." systemctl restart kubelet {{- end }} -{{- if (and .Values.gke.enabled (or .Values.masquerade .Values.gke.disableDefaultSnat))}} +{{- if (and .Values.gke.enabled (or .Values.enableIPv4Masquerade .Values.gke.disableDefaultSnat))}} # If Cilium is configured to manage masquerading of traffic leaving the node, # we need to disable the IP-MASQ chain because even if ip-masq-agent # is not installed, the node init script installs some default rules into @@ -118,7 +120,8 @@ iptables -w -t nat -D POSTROUTING -m comment --comment "ip-masq: ensure nat POST {{- end }} {{- if not (eq .Values.nodeinit.bootstrapFile "") }} -date > {{ .Values.nodeinit.bootstrapFile }} +mkdir -p {{ .Values.nodeinit.bootstrapFile | dir | quote }} +date > {{ .Values.nodeinit.bootstrapFile | quote }} {{- end }} {{- if .Values.azure.enabled }} diff --git a/helm/cilium/templates/NOTES.txt b/helm/cilium/templates/NOTES.txt index 1730b66..f540507 100644 --- a/helm/cilium/templates/NOTES.txt +++ b/helm/cilium/templates/NOTES.txt @@ -17,6 +17,6 @@ You have successfully installed {{ title .Chart.Name }}. {{- end }} -Your release version is {{ .Chart.AppVersion }}. +Your release version is {{ .Chart.Version }}. -For any further help, visit https://docs.cilium.io/en/v{{ (semver .Chart.AppVersion).Major }}.{{ (semver .Chart.AppVersion).Minor }}/gettinghelp +For any further help, visit https://docs.cilium.io/en/v{{ (semver .Chart.Version).Major }}.{{ (semver .Chart.Version).Minor }}/gettinghelp diff --git a/helm/cilium/templates/_helpers.tpl b/helm/cilium/templates/_helpers.tpl index b5911cf..e0eb6ec 100644 --- a/helm/cilium/templates/_helpers.tpl +++ b/helm/cilium/templates/_helpers.tpl @@ -19,8 +19,12 @@ will return `quay.io/cilium/cilium:v1.10.1@abcdefgh` */}} {{- define "cilium.image" -}} {{- $digest := (.useDigest | default false) | ternary (printf "@%s" .digest) "" -}} +{{- if .override -}} +{{- printf "%s" .override -}} +{{- else -}} {{- printf "%s:%s%s" .repository .tag $digest -}} {{- end -}} +{{- end -}} {{/* Return user specify priorityClass or default criticalPriorityClass diff --git a/helm/cilium/templates/cilium-agent/clusterrole.yaml b/helm/cilium/templates/cilium-agent/clusterrole.yaml index e3be780..fceb2ec 100644 --- a/helm/cilium/templates/cilium-agent/clusterrole.yaml +++ b/helm/cilium/templates/cilium-agent/clusterrole.yaml @@ -28,39 +28,22 @@ rules: resources: - namespaces - services - - nodes - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - pods - - pods/finalizers - verbs: - - get - - list - - watch - - update - - delete -- apiGroups: - - "" - resources: + - endpoints - nodes verbs: - get - list - watch - - update +{{- if .Values.annotateK8sNode }} - apiGroups: - "" resources: - - nodes - nodes/status verbs: + # To annotate the k8s node with Cilium's metadata - patch +{{- end }} - apiGroups: - apiextensions.k8s.io resources: @@ -89,21 +72,15 @@ rules: resources: - ciliumnetworkpolicies - ciliumnetworkpolicies/status - - ciliumnetworkpolicies/finalizers - ciliumclusterwidenetworkpolicies - ciliumclusterwidenetworkpolicies/status - - ciliumclusterwidenetworkpolicies/finalizers - ciliumendpoints - ciliumendpoints/status - - ciliumendpoints/finalizers - ciliumnodes - ciliumnodes/status - - ciliumnodes/finalizers - ciliumidentities - - ciliumidentities/finalizers - ciliumlocalredirectpolicies - ciliumlocalredirectpolicies/status - - ciliumlocalredirectpolicies/finalizers - ciliumegressnatpolicies - ciliumendpointslices verbs: diff --git a/helm/cilium/templates/cilium-agent/daemonset.yaml b/helm/cilium/templates/cilium-agent/daemonset.yaml index cd96196..6513f6f 100644 --- a/helm/cilium/templates/cilium-agent/daemonset.yaml +++ b/helm/cilium/templates/cilium-agent/daemonset.yaml @@ -7,6 +7,9 @@ {{- if semverCompare ">=1.8" (default "1.8" .Values.upgradeCompatibility) -}} {{- $defaultKeepDeprecatedProbes = false -}} {{- end -}} + +{{- $kubeProxyReplacement := (coalesce .Values.kubeProxyReplacement "disabled") -}} + --- apiVersion: apps/v1 kind: DaemonSet @@ -43,11 +46,6 @@ spec: # ensure pods roll when configmap updates cilium.io/cilium-configmap-checksum: {{ include (print $.Template.BasePath "/cilium-configmap.yaml") . | sha256sum | quote }} {{- end }} - # This annotation plus the CriticalAddonsOnly toleration makes - # cilium to be a critical pod in the cluster, which ensures cilium - # gets priority scheduling. - # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - scheduler.alpha.kubernetes.io/critical-pod: "" {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -221,6 +219,12 @@ spec: {{- end }} {{- if or .Values.prometheus.enabled .Values.hubble.metrics.enabled }} ports: + {{- if .Values.hubble.peerService.enabled }} + - name: peer-service + containerPort: {{ .Values.hubble.peerService.targetPort }} + hostPort: {{ .Values.hubble.peerService.targetPort }} + protocol: TCP + {{- end }} {{- if .Values.prometheus.enabled }} - name: prometheus containerPort: {{ .Values.prometheus.port }} @@ -242,6 +246,7 @@ spec: {{- end }} securityContext: privileged: true + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: {{- /* CRI-O already mounts the BPF filesystem */ -}} {{- if not (eq .Values.containerRuntime.integration "crio") }} @@ -329,6 +334,7 @@ spec: {{- range $type := .Values.monitor.eventTypes }} - --type={{ $type }} {{- end }} + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: cilium-run mountPath: /var/run/cilium @@ -341,6 +347,8 @@ spec: # In managed etcd mode, Cilium must be able to resolve the DNS name of # the etcd service dnsPolicy: ClusterFirstWithHostNet + {{- else if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy }} {{- end }} hostNetwork: true initContainers: @@ -372,6 +380,33 @@ spec: mountPath: /hostproc - name: cni-path mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + - name: apply-sysctl-overwrites + image: {{ include "cilium.image" .Values.image | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: BIN_PATH + value: {{ .Values.cni.binPath }} + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError securityContext: privileged: true {{- end }} @@ -383,13 +418,14 @@ spec: - sh - -c - | - until test -f {{ .Values.nodeinit.bootstrapFile | quote }}; do + until test -s {{ (print "/tmp/cilium-bootstrap.d/" (.Values.nodeinit.bootstrapFile | base)) | quote }}; do echo "Waiting on node-init to run..."; sleep 1; done + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - - name: cilium-bootstrap-file - mountPath: {{ .Values.nodeinit.bootstrapFile }} + - name: cilium-bootstrap-file-dir + mountPath: "/tmp/cilium-bootstrap.d" {{- end }} - name: clean-cilium-state image: {{ include "cilium.image" .Values.image | quote }} @@ -420,6 +456,7 @@ spec: {{- with .Values.extraEnv }} {{- toYaml . | nindent 8 }} {{- end }} + terminationMessagePolicy: FallbackToLogsOnError securityContext: privileged: true volumeMounts: @@ -438,6 +475,39 @@ spec: resources: {{- toYaml . | trim | nindent 10 }} {{- end }} + {{- if and .Values.waitForKubeProxy (ne $kubeProxyReplacement "strict") }} + - name: wait-for-kube-proxy + image: {{ include "cilium.image" .Values.image | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + privileged: true + command: + - bash + - -c + - | + while true + do + if iptables-nft-save -t mangle | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-PROXY-CANARY)'; then + echo "Found KUBE-IPTABLES-HINT or KUBE-PROXY-CANARY iptables rule in 'iptables-nft-save -t mangle'" + exit 0 + fi + if ip6tables-nft-save -t mangle | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-PROXY-CANARY)'; then + echo "Found KUBE-IPTABLES-HINT or KUBE-PROXY-CANARY iptables rule in 'ip6tables-nft-save -t mangle'" + exit 0 + fi + if iptables-legacy-save | grep -E '^:KUBE-PROXY-CANARY'; then + echo "Found KUBE-PROXY-CANARY iptables rule in 'iptables-legacy-save" + exit 0 + fi + if ip6tables-legacy-save | grep -E '^:KUBE-PROXY-CANARY'; then + echo "KUBE-PROXY-CANARY iptables rule in 'ip6tables-legacy-save'" + exit 0 + fi + echo "Waiting for kube-proxy to create iptables rules..."; + sleep 1; + done + terminationMessagePolicy: FallbackToLogsOnError + {{- end }} # wait-for-kube-proxy {{ if and (.Values.cleanupKubeProxy) (not (eq .Values.kubeProxyReplacement "disabled")) }} - name: cleanup-kube-proxy-iptables image: {{ include "cilium.image" .Values.image | quote }} @@ -459,6 +529,15 @@ spec: tolerations: {{- toYaml . | trim | nindent 8 }} {{- end }} + {{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.config.enabled }} + hostAliases: + {{- range $cluster := .Values.clustermesh.config.clusters }} + {{- range $ip := $cluster.ips }} + - ip: {{ $ip }} + hostnames: [ "{{ $cluster.name }}.{{ $.Values.clustermesh.config.domain }}" ] + {{- end }} + {{- end }} + {{- end }} volumes: # To keep state between restarts / upgrades - name: cilium-run @@ -511,10 +590,10 @@ spec: type: FileOrCreate {{- end }} {{- if and .Values.nodeinit.enabled .Values.nodeinit.bootstrapFile }} - - name: cilium-bootstrap-file + - name: cilium-bootstrap-file-dir hostPath: - path: {{ .Values.nodeinit.bootstrapFile }} - type: FileOrCreate + path: {{ .Values.nodeinit.bootstrapFile | dir | quote }} + type: DirectoryOrCreate {{- end }} {{- range .Values.extraHostPathMounts }} - name: {{ .name }} diff --git a/helm/cilium/templates/cilium-agent/podsecuritypolicy.yaml b/helm/cilium/templates/cilium-agent/podsecuritypolicy.yaml index 2580136..088eb59 100644 --- a/helm/cilium/templates/cilium-agent/podsecuritypolicy.yaml +++ b/helm/cilium/templates/cilium-agent/podsecuritypolicy.yaml @@ -19,8 +19,16 @@ spec: hostNetwork: true hostIPC: false hostPID: false - {{- if .Values.prometheus.enabled }} hostPorts: + {{- if .Values.hubble.peerService.enabled }} + - min: {{ .Values.hubble.peerService.targetPort }} + max: {{ .Values.hubble.peerService.targetPort }} + {{- end }} + {{- if .Values.hubble.metrics.enabled }} + - min: {{ .Values.hubble.metrics.port }} + max: {{ .Values.hubble.metrics.port }} + {{- end }} + {{- if .Values.prometheus.enabled }} - min: {{ .Values.prometheus.port }} max: {{ .Values.prometheus.port }} - min: {{ .Values.proxy.prometheus.port }} diff --git a/helm/cilium/templates/cilium-configmap.yaml b/helm/cilium/templates/cilium-configmap.yaml index d3c05cf..5ed41c3 100644 --- a/helm/cilium/templates/cilium-configmap.yaml +++ b/helm/cilium/templates/cilium-configmap.yaml @@ -109,6 +109,10 @@ data: cilium-endpoint-gc-interval: "{{ .Values.operator.endpointGCInterval }}" {{- end }} +{{- if hasKey .Values.operator "nodeGCInterval" }} + nodes-gc-interval: "{{ .Values.operator.nodeGCInterval | default "0s" }}" +{{- end }} + {{- if hasKey .Values "disableEndpointCRD" }} # Disable the usage of CiliumEndpoint CRD disable-endpoint-crd: "{{ .Values.disableEndpointCRD }}" @@ -135,7 +139,7 @@ data: debug-verbose: "{{ .Values.debug.verbose }}" {{- end }} -{{- if ne (int .Values.healthPort) 9876 }} +{{- if ne (int .Values.healthPort) 9879 }} # Set the TCP port for the agent health status API. This is not the port used # for cilium-health. agent-health-port: "{{ .Values.healthPort }}" @@ -356,8 +360,10 @@ data: tunnel: "disabled" enable-endpoint-routes: "true" enable-local-node-route: "false" +{{- else if .Values.aksbyocni.enabled }} + tunnel: "vxlan" {{- else }} - tunnel: {{ .Values.tunnel }} + tunnel: {{ .Values.tunnel | quote }} {{- end }} {{- if hasKey .Values "tunnelPort" }} @@ -708,7 +714,11 @@ data: # A space separated list of iptables chains to disable when installing feeder rules. disable-iptables-feeder-rules: {{ .Values.disableIptablesFeederRules | join " " | quote }} {{- end }} +{{- if .Values.aksbyocni.enabled }} + ipam: "cluster-pool" +{{- else }} ipam: {{ $ipam | quote }} +{{- end }} {{- if eq $ipam "cluster-pool" }} {{- if .Values.ipv4.enabled }} @@ -792,7 +802,29 @@ data: enable-k8s-terminating-endpoint: {{ .Values.enableK8sTerminatingEndpoint | quote }} {{- end }} +{{- if .Values.annotateK8sNode }} + annotate-k8s-node: "true" +{{- end }} + +{{- if .Values.operator.removeNodeTaints }} + remove-cilium-node-taints: "true" +{{- end }} +{{- if .Values.operator.setNodeNetworkStatus }} + set-cilium-is-up-condition: "true" +{{- end }} + +{{- if .Values.operator.unmanagedPodWatcher.restart }} + unmanaged-pod-watcher-interval: {{ .Values.operator.unmanagedPodWatcher.intervalSeconds | quote }} +{{- else }} + unmanaged-pod-watcher-interval: "0" +{{- end }} + {{- if .Values.extraConfig }} {{ toYaml .Values.extraConfig | nindent 2 }} {{- end }} + +{{- if hasKey .Values "agentNotReadyTaintKey" }} + agent-not-ready-taint-key: {{ .Values.agentNotReadyTaintKey | quote }} +{{- end }} + {{- end }} diff --git a/helm/cilium/templates/cilium-nodeinit/daemonset.yaml b/helm/cilium/templates/cilium-nodeinit/daemonset.yaml index f2411a0..fb7e122 100644 --- a/helm/cilium/templates/cilium-nodeinit/daemonset.yaml +++ b/helm/cilium/templates/cilium-nodeinit/daemonset.yaml @@ -34,27 +34,21 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - volumes: - # To access iptables concurrently with other processes (e.g. kube-proxy) - - hostPath: - path: /run/xtables.lock - type: FileOrCreate - name: xtables-lock containers: - name: node-init image: {{ include "cilium.image" .Values.nodeinit.image | quote }} imagePullPolicy: {{ .Values.nodeinit.image.pullPolicy }} securityContext: privileged: true - volumeMounts: - # To access iptables concurrently with other processes (e.g. kube-proxy) - - mountPath: /run/xtables.lock - name: xtables-lock lifecycle: {{- if .Values.eni.enabled }} postStart: exec: command: + - nsenter + - --target=1 + - --mount + - -- - "/bin/sh" - "-c" - | @@ -74,8 +68,6 @@ spec: {{- tpl (.Files.Get "files/nodeinit/prestop.bash") . | nindent 20 }} {{- end }} env: - - name: CHECKPOINT_PATH - value: /tmp/node-init.cilium.io # STARTUP_SCRIPT is the script run on node bootstrap. Node # bootstrapping can be customized in this script. This script is invoked # using nsenter, so it runs in the host's network and mount namespace using @@ -83,4 +75,5 @@ spec: - name: STARTUP_SCRIPT value: | {{- tpl (.Files.Get "files/nodeinit/startup.bash") . | nindent 14 }} + terminationMessagePolicy: FallbackToLogsOnError {{- end }} diff --git a/helm/cilium/templates/cilium-operator/_helpers.tpl b/helm/cilium/templates/cilium-operator/_helpers.tpl index 8334196..0910de6 100644 --- a/helm/cilium/templates/cilium-operator/_helpers.tpl +++ b/helm/cilium/templates/cilium-operator/_helpers.tpl @@ -26,7 +26,11 @@ Return cilium operator image */}} {{- define "cilium.operator.image" -}} +{{- if .Values.operator.image.override -}} +{{- printf "%s" .Values.operator.image.override -}} +{{- else -}} {{- $cloud := include "cilium.operator.cloud" . }} {{- $imageDigest := include "cilium.operator.imageDigestName" . }} {{- printf "%s-%s%s:%s%s" .Values.operator.image.repository $cloud .Values.operator.image.suffix .Values.operator.image.tag $imageDigest -}} {{- end -}} +{{- end -}} diff --git a/helm/cilium/templates/cilium-operator/clusterrole.yaml b/helm/cilium/templates/cilium-operator/clusterrole.yaml index 9665c3f..ec2b917 100644 --- a/helm/cilium/templates/cilium-operator/clusterrole.yaml +++ b/helm/cilium/templates/cilium-operator/clusterrole.yaml @@ -7,14 +7,44 @@ rules: - apiGroups: - "" resources: - # to automatically delete [core|kube]dns pods so that are starting to being - # managed by Cilium - pods verbs: - get - list - watch +{{- if hasKey .Values "disableEndpointCRD" }} +{{- if (eq (.Values.disableEndpointCRD | quote ) ( "false" | quote )) }} +{{- if (and .Values.operator.unmanagedPodWatcher.restart (ne (.Values.operator.unmanagedPodWatcher.intervalSeconds | int64) 0 ) ) }} + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium - delete +{{- end }} +{{- end }} +{{- end }} +{{- if or .Values.operator.removeNodeTaints .Values.operator.setNodeNetworkStatus .Values.operator.endpointGCInterval }} +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +{{- end }} +{{- if or .Values.operator.removeNodeTaints .Values.operator.setNodeNetworkStatus }} +- apiGroups: + - "" + resources: +{{- if .Values.operator.removeNodeTaints }} + # To remove node taints + - nodes +{{- end }} +{{- if .Values.operator.setNodeNetworkStatus }} + # To set NetworkUnavailable false on startup + - nodes/status +{{- end }} + verbs: + - patch +{{- end }} - apiGroups: - discovery.k8s.io resources: diff --git a/helm/cilium/templates/cilium-operator/deployment.yaml b/helm/cilium/templates/cilium-operator/deployment.yaml index f6e77c8..e88a8de 100644 --- a/helm/cilium/templates/cilium-operator/deployment.yaml +++ b/helm/cilium/templates/cilium-operator/deployment.yaml @@ -147,7 +147,7 @@ spec: {{- range $key, $value := .Values.operator.extraEnv }} - name: {{ $key }} value: {{ $value }} - {{- end }} + {{- end }} {{- if .Values.operator.prometheus.enabled }} ports: - name: prometheus @@ -200,11 +200,14 @@ spec: resources: {{- toYaml . | trim | nindent 10 }} {{- end }} + terminationMessagePolicy: FallbackToLogsOnError hostNetwork: true {{- if and .Values.etcd.managed (not .Values.etcd.k8sService) }} # In managed etcd mode, Cilium must be able to resolve the DNS name of # the etcd service dnsPolicy: ClusterFirstWithHostNet + {{- else if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.operator.dnsPolicy }} {{- end }} restartPolicy: Always priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.operator.priorityClassName "system-cluster-critical") }} diff --git a/helm/cilium/templates/cilium-preflight/clusterrole.yaml b/helm/cilium/templates/cilium-preflight/clusterrole.yaml index 3964d8b..6fd304d 100644 --- a/helm/cilium/templates/cilium-preflight/clusterrole.yaml +++ b/helm/cilium/templates/cilium-preflight/clusterrole.yaml @@ -28,39 +28,22 @@ rules: resources: - namespaces - services - - nodes - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - pods - - pods/finalizers - verbs: - - get - - list - - watch - - update - - delete -- apiGroups: - - "" - resources: + - endpoints - nodes verbs: - get - list - watch - - update +{{- if .Values.annotateK8sNode }} - apiGroups: - "" resources: - - nodes - nodes/status verbs: + # To annotate the k8s node with Cilium's metadata - patch +{{- end }} - apiGroups: - apiextensions.k8s.io resources: @@ -89,21 +72,15 @@ rules: resources: - ciliumnetworkpolicies - ciliumnetworkpolicies/status - - ciliumnetworkpolicies/finalizers - ciliumclusterwidenetworkpolicies - ciliumclusterwidenetworkpolicies/status - - ciliumclusterwidenetworkpolicies/finalizers - ciliumendpoints - ciliumendpoints/status - - ciliumendpoints/finalizers - ciliumnodes - ciliumnodes/status - - ciliumnodes/finalizers - ciliumidentities - - ciliumidentities/finalizers - ciliumlocalredirectpolicies - ciliumlocalredirectpolicies/status - - ciliumlocalredirectpolicies/finalizers - ciliumegressnatpolicies - ciliumendpointslices verbs: diff --git a/helm/cilium/templates/cilium-preflight/clusterrolebinding.yaml b/helm/cilium/templates/cilium-preflight/clusterrolebinding.yaml index b4a6d54..fbaf6de 100644 --- a/helm/cilium/templates/cilium-preflight/clusterrolebinding.yaml +++ b/helm/cilium/templates/cilium-preflight/clusterrolebinding.yaml @@ -9,6 +9,6 @@ roleRef: name: cilium-pre-flight subjects: - kind: ServiceAccount - name: {{ .Values.serviceAccounts.preflight.name | quote }} + name: {{ .Values.serviceAccounts.preflight.name | quote }} namespace: {{ .Release.Namespace }} {{- end }} diff --git a/helm/cilium/templates/cilium-preflight/daemonset.yaml b/helm/cilium/templates/cilium-preflight/daemonset.yaml index 27c7611..aafbc2e 100644 --- a/helm/cilium/templates/cilium-preflight/daemonset.yaml +++ b/helm/cilium/templates/cilium-preflight/daemonset.yaml @@ -33,6 +33,7 @@ spec: command: ["/bin/echo"] args: - "hello" + terminationMessagePolicy: FallbackToLogsOnError containers: - name: cilium-pre-flight-check image: {{ include "cilium.image" .Values.preflight.image | quote }} @@ -68,6 +69,7 @@ spec: readOnly: true {{- end }} {{- end }} + terminationMessagePolicy: FallbackToLogsOnError {{- if ne .Values.preflight.tofqdnsPreCache "" }} - name: cilium-pre-flight-fqdn-precache image: {{ include "cilium.image" .Values.preflight.image | quote }} @@ -115,6 +117,7 @@ spec: readOnly: true {{- end }} {{- end }} + terminationMessagePolicy: FallbackToLogsOnError {{- end }} hostNetwork: true # This is here to seamlessly allow migrate-identity to work with diff --git a/helm/cilium/templates/cilium-preflight/deployment.yaml b/helm/cilium/templates/cilium-preflight/deployment.yaml index ef04145..6dcc41b 100644 --- a/helm/cilium/templates/cilium-preflight/deployment.yaml +++ b/helm/cilium/templates/cilium-preflight/deployment.yaml @@ -70,6 +70,7 @@ spec: - name: KUBERNETES_SERVICE_PORT value: {{ .Values.k8sServicePort | quote }} {{- end }} + terminationMessagePolicy: FallbackToLogsOnError hostNetwork: true restartPolicy: Always priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.preflight.priorityClassName "system-cluster-critical") }} diff --git a/helm/cilium/templates/clustermesh-apiserver/deployment.yaml b/helm/cilium/templates/clustermesh-apiserver/deployment.yaml index f47eb51..24210ce 100644 --- a/helm/cilium/templates/clustermesh-apiserver/deployment.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/deployment.yaml @@ -67,6 +67,7 @@ spec: volumeMounts: - name: etcd-data-dir mountPath: /var/run/etcd + terminationMessagePolicy: FallbackToLogsOnError containers: - name: etcd image: {{ include "cilium.image" .Values.clustermesh.apiserver.etcd.image | quote }} @@ -97,6 +98,7 @@ spec: readOnly: true - name: etcd-data-dir mountPath: /var/run/etcd + terminationMessagePolicy: FallbackToLogsOnError - name: apiserver image: {{ include "cilium.image" .Values.clustermesh.apiserver.image | quote }} imagePullPolicy: {{ .Values.clustermesh.apiserver.image.pullPolicy }} @@ -107,6 +109,7 @@ spec: - --debug {{- end }} - --cluster-name=$(CLUSTER_NAME) + - --cluster-id=$(CLUSTER_ID) - --kvstore-opt - etcd.config=/var/lib/cilium/etcd-config.yaml env: @@ -126,6 +129,12 @@ spec: configMapKeyRef: name: cilium-config key: identity-allocation-mode + - name: ENABLE_K8S_ENDPOINT_SLICE + valueFrom: + configMapKeyRef: + name: cilium-config + key: enable-k8s-endpoint-slice + optional: true {{- with .Values.clustermesh.apiserver.resources }} resources: {{- toYaml . | nindent 10 }} @@ -134,6 +143,7 @@ spec: - name: etcd-admin-client mountPath: /var/lib/cilium/etcd-secrets readOnly: true + terminationMessagePolicy: FallbackToLogsOnError volumes: - name: etcd-server-secrets secret: diff --git a/helm/cilium/templates/clustermesh-config/_helpers.tpl b/helm/cilium/templates/clustermesh-config/_helpers.tpl new file mode 100644 index 0000000..e2e66dc --- /dev/null +++ b/helm/cilium/templates/clustermesh-config/_helpers.tpl @@ -0,0 +1,14 @@ +{{- define "clustermesh-config-generate-etcd-cfg" }} +{{- $cluster := index . 0 -}} +{{- $domain := index . 1 -}} + +endpoints: +{{- if $cluster.ips }} +- https://{{ $cluster.name }}.{{ $domain }}:{{ $cluster.port }} +{{ else }} +- https://{{ $cluster.address | required "missing clustermesh.apiserver.config.clusters.address" }}:{{ $cluster.port }} +{{- end }} +trusted-ca-file: /var/lib/cilium/clustermesh/{{ $cluster.name }}.etcd-client-ca.crt +key-file: /var/lib/cilium/clustermesh/{{ $cluster.name }}.etcd-client.key +cert-file: /var/lib/cilium/clustermesh/{{ $cluster.name }}.etcd-client.crt +{{- end }} diff --git a/helm/cilium/templates/clustermesh-config/clustermesh-secret.yaml b/helm/cilium/templates/clustermesh-config/clustermesh-secret.yaml new file mode 100644 index 0000000..1e34def --- /dev/null +++ b/helm/cilium/templates/clustermesh-config/clustermesh-secret.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.config.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: cilium-clustermesh + namespace: {{ .Release.Namespace }} +data: + {{- range .Values.clustermesh.config.clusters }} + {{ .name }}: {{ include "clustermesh-config-generate-etcd-cfg" (list . $.Values.clustermesh.config.domain) | b64enc }} + {{ .name }}.etcd-client-ca.crt: {{ $.Values.clustermesh.apiserver.tls.ca.cert }} + {{ .name }}.etcd-client.key: {{ .tls.key }} + {{ .name }}.etcd-client.crt: {{ .tls.cert }} + {{- end }} +{{- end }} diff --git a/helm/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml b/helm/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml index 8761494..a4e5d41 100644 --- a/helm/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml +++ b/helm/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml @@ -73,6 +73,7 @@ spec: image: {{ .Values.etcd.image.repository }}:{{ .Values.etcd.image.tag }} imagePullPolicy: {{ .Values.etcd.image.pullPolicy }} name: cilium-etcd-operator + terminationMessagePolicy: FallbackToLogsOnError dnsPolicy: ClusterFirst hostNetwork: true priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.clustermesh.apiserver.priorityClassName "system-cluster-critical") }} diff --git a/helm/cilium/templates/hubble-relay/configmap.yaml b/helm/cilium/templates/hubble-relay/configmap.yaml index 24da88f..6287157 100644 --- a/helm/cilium/templates/hubble-relay/configmap.yaml +++ b/helm/cilium/templates/hubble-relay/configmap.yaml @@ -1,4 +1,8 @@ {{- if and .Values.hubble.enabled .Values.hubble.relay.enabled }} +{{- $peerSvcPort := .Values.hubble.peerService.servicePort -}} +{{- if not .Values.hubble.peerService.servicePort }} +{{- $peerSvcPort = (.Values.hubble.tls.enabled | ternary 443 80) -}} +{{- end }} --- apiVersion: v1 kind: ConfigMap @@ -7,7 +11,12 @@ metadata: namespace: {{ .Release.Namespace }} data: config.yaml: | + cluster-name: {{ .Values.cluster.name }} + {{- if and .Values.hubble.enabled .Values.hubble.peerService.enabled }} + peer-service: "hubble-peer.{{ .Release.Namespace }}.svc.{{ .Values.hubble.peerService.clusterDomain }}:{{ $peerSvcPort }}" + {{- else }} peer-service: unix://{{ .Values.hubble.socketPath }} + {{- end }} listen-address: {{ .Values.hubble.relay.listenHost }}:{{ .Values.hubble.relay.listenPort }} dial-timeout: {{ .Values.hubble.relay.dialTimeout }} retry-timeout: {{ .Values.hubble.relay.retryTimeout }} diff --git a/helm/cilium/templates/hubble-relay/deployment.yaml b/helm/cilium/templates/hubble-relay/deployment.yaml index d2ff2c0..aaee9b3 100644 --- a/helm/cilium/templates/hubble-relay/deployment.yaml +++ b/helm/cilium/templates/hubble-relay/deployment.yaml @@ -1,4 +1,5 @@ {{- if and .Values.hubble.enabled .Values.hubble.relay.enabled }} +{{- $mountSocket := not .Values.hubble.peerService.enabled -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -70,9 +71,11 @@ spec: {{- toYaml . | trim | nindent 12 }} {{- end }} volumeMounts: + {{- if $mountSocket }} - name: hubble-sock-dir mountPath: {{ dir .Values.hubble.socketPath }} readOnly: true + {{- end }} - name: config mountPath: /etc/hubble-relay readOnly: true @@ -81,6 +84,7 @@ spec: mountPath: /var/lib/hubble-relay/tls readOnly: true {{- end }} + terminationMessagePolicy: FallbackToLogsOnError restartPolicy: Always priorityClassName: {{ .Values.hubble.relay.priorityClassName }} serviceAccount: {{ .Values.serviceAccounts.relay.name | quote }} @@ -102,10 +106,12 @@ spec: items: - key: config.yaml path: config.yaml + {{- if $mountSocket }} - name: hubble-sock-dir hostPath: path: {{ dir .Values.hubble.socketPath }} type: Directory + {{- end }} {{- if .Values.hubble.tls.enabled }} - name: tls projected: diff --git a/helm/cilium/templates/hubble-ui/_nginx.tpl b/helm/cilium/templates/hubble-ui/_nginx.tpl new file mode 100644 index 0000000..d3501d6 --- /dev/null +++ b/helm/cilium/templates/hubble-ui/_nginx.tpl @@ -0,0 +1,39 @@ +{{- define "hubble-ui.nginx.conf" }} +server { + listen 8081; +{{- if .Values.hubble.ui.frontend.server.ipv6.enabled }} + listen [::]:8081; +{{- end }} + server_name localhost; + root /app; + index index.html; + client_max_body_size 1G; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + + # CORS + add_header Access-Control-Allow-Methods "GET, POST, PUT, HEAD, DELETE, OPTIONS"; + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Max-Age 1728000; + add_header Access-Control-Expose-Headers content-length,grpc-status,grpc-message; + add_header Access-Control-Allow-Headers range,keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout; + if ($request_method = OPTIONS) { + return 204; + } + # /CORS + + location /api { + proxy_http_version 1.1; + proxy_pass_request_headers on; + proxy_hide_header Access-Control-Allow-Origin; + proxy_pass http://127.0.0.1:8090; + } + + location / { + try_files $uri $uri/ /index.html; + } + } +} +{{- end }} diff --git a/helm/cilium/templates/hubble-ui/configmap.yaml b/helm/cilium/templates/hubble-ui/configmap.yaml index ccfebe0..bbab253 100644 --- a/helm/cilium/templates/hubble-ui/configmap.yaml +++ b/helm/cilium/templates/hubble-ui/configmap.yaml @@ -3,8 +3,8 @@ apiVersion: v1 kind: ConfigMap metadata: - name: hubble-ui-envoy + name: hubble-ui-nginx namespace: {{ .Release.Namespace }} data: - {{ (.Files.Glob "files/envoy/*").AsConfig | nindent 2 }} + nginx.conf: {{ include "hubble-ui.nginx.conf" . | trim | quote }} {{- end }} diff --git a/helm/cilium/templates/hubble-ui/deployment.yaml b/helm/cilium/templates/hubble-ui/deployment.yaml index 593d10e..0793be2 100644 --- a/helm/cilium/templates/hubble-ui/deployment.yaml +++ b/helm/cilium/templates/hubble-ui/deployment.yaml @@ -16,7 +16,7 @@ spec: annotations: {{- if .Values.hubble.ui.rollOutPods }} # ensure pods roll when configmap updates - cilium.io/hubble-ui-envoy-configmap-checksum: {{ include (print $.Template.BasePath "/hubble-ui/configmap.yaml") . | sha256sum | quote }} + cilium.io/hubble-ui-nginx-configmap-checksum: {{ include (print $.Template.BasePath "/hubble-ui/configmap.yaml") . | sha256sum | quote }} {{- end }} {{- with .Values.hubble.ui.podAnnotations }} {{- toYaml . | nindent 8 }} @@ -30,6 +30,8 @@ spec: {{- if .Values.hubble.ui.securityContext.enabled }} securityContext: runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 {{- end }} priorityClassName: {{ .Values.hubble.ui.priorityClassName }} serviceAccount: {{ .Values.serviceAccounts.ui.name | quote }} @@ -52,11 +54,16 @@ spec: imagePullPolicy: {{ .Values.hubble.ui.frontend.image.pullPolicy }} ports: - name: http - containerPort: 8080 + containerPort: 8081 {{- with .Values.hubble.ui.frontend.resources }} resources: {{- toYaml . | trim | nindent 10 }} {{- end }} + volumeMounts: + - name: hubble-ui-nginx-conf + mountPath: /etc/nginx/conf.d/default.conf + subPath: nginx.conf + terminationMessagePolicy: FallbackToLogsOnError - name: backend image: {{ include "cilium.image" .Values.hubble.ui.backend.image | quote }} imagePullPolicy: {{ .Values.hubble.ui.backend.image.pullPolicy }} @@ -93,30 +100,12 @@ spec: mountPath: /var/lib/hubble-ui/certs readOnly: true {{- end }} - - name: proxy - image: {{ include "cilium.image" .Values.hubble.ui.proxy.image | quote }} - imagePullPolicy: {{ .Values.hubble.ui.proxy.image.pullPolicy }} - ports: - - name: http - containerPort: 8081 - {{- with .Values.hubble.ui.proxy.resources }} - resources: - {{- toYaml . | trim | nindent 10 }} - {{- end }} - command: [envoy] - args: - - -c - - /etc/envoy.yaml - - -l - - info - volumeMounts: - - name: hubble-ui-envoy-yaml - mountPath: /etc/envoy.yaml - subPath: envoy.yaml + terminationMessagePolicy: FallbackToLogsOnError volumes: - - name: hubble-ui-envoy-yaml - configMap: - name: hubble-ui-envoy + - configMap: + defaultMode: 420 + name: hubble-ui-nginx + name: hubble-ui-nginx-conf {{- if .Values.hubble.relay.tls.server.enabled }} - name: hubble-ui-client-certs {{- if .Values.hubble.ui.standalone.enabled }} diff --git a/helm/cilium/templates/hubble/peer-service.yaml b/helm/cilium/templates/hubble/peer-service.yaml new file mode 100644 index 0000000..698e6bf --- /dev/null +++ b/helm/cilium/templates/hubble/peer-service.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.agent .Values.hubble.enabled .Values.hubble.peerService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: hubble-peer + namespace: {{ .Release.Namespace }} + labels: + k8s-app: cilium +spec: + selector: + k8s-app: cilium + ports: + - name: peer-service + {{- if .Values.hubble.peerService.servicePort }} + port: {{ .Values.hubble.peerService.servicePort }} + {{- else }} + port: {{ .Values.hubble.tls.enabled | ternary 443 80 }} + {{- end }} + protocol: TCP + targetPort: {{ .Values.hubble.peerService.targetPort }} +{{- if semverCompare ">=1.22-0" .Capabilities.KubeVersion.GitVersion }} + internalTrafficPolicy: Local +{{- end }} +{{- end }} diff --git a/helm/cilium/templates/validate.yaml b/helm/cilium/templates/validate.yaml index c25cdcd..7bf77f1 100644 --- a/helm/cilium/templates/validate.yaml +++ b/helm/cilium/templates/validate.yaml @@ -25,11 +25,21 @@ {{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "certmanager") }} {{- if not .Values.hubble.tls.auto.certManagerIssuerRef }} {{ fail "Hubble TLS certgen method=certmanager requires user specify .Values.hubble.tls.auto.certManagerIssuerRef" }} - {{- end }} + {{- end }} {{- end }} {{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "certmanager") }} {{- if not .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef }} {{ fail "ClusterMesh TLS certgen method=certmanager requires user specify .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef" }} - {{- end }} + {{- end }} +{{- end }} + +{{/* validate hubble-ui specific config */}} +{{- if .Values.hubble.ui.enabled }} + {{- if regexReplaceAll "@.*$" .Values.hubble.ui.backend.image.tag "" | trimPrefix "v" | semverCompare "<0.9.0" }} + {{ fail "Hubble UI requires hubble.ui.backend.image.tag to be '>=v0.9.0'" }} + {{- end }} + {{- if regexReplaceAll "@.*$" .Values.hubble.ui.frontend.image.tag "" | trimPrefix "v" | semverCompare "<0.9.0" }} + {{ fail "Hubble UI requires hubble.ui.frontend.image.tag to be '>=v0.9.0'" }} + {{- end }} {{- end }} diff --git a/helm/cilium/values.yaml b/helm/cilium/values.yaml index db7ef95..e071b92 100644 --- a/helm/cilium/values.yaml +++ b/helm/cilium/values.yaml @@ -121,6 +121,10 @@ affinity: # -- The priority class to use for cilium-agent. priorityClassName: "" +# -- DNS policy for Cilium agent pods. +# Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy +dnsPolicy: "" + # -- Additional agent container arguments. extraArgs: [] @@ -198,12 +202,23 @@ updateStrategy: # Configuration Values for cilium-agent +aksbyocni: + # -- Enable AKS BYOCNI integration. + # Note that this is incompatible with AKS clusters not created in BYOCNI mode: + # use Azure integration (`azure.enabled`) instead. + enabled: false + # -- Enable installation of PodCIDR routes between worker # nodes if worker nodes share a common L2 network segment. autoDirectNodeRoutes: false +# -- Annotate k8s node upon initialization with Cilium's metadata. +annotateK8sNode: true + azure: - # -- Enable Azure integration + # -- Enable Azure integration. + # Note that this is incompatible with AKS clusters created in BYOCNI mode: use + # AKS BYOCNI integration (`aksbyocni.enabled`) instead. enabled: false # resourceGroup: group1 # subscriptionID: 00000000-0000-0000-0000-000000000000 @@ -299,7 +314,7 @@ bpf: # -- Configure explicitly allowed VLAN id's for bpf logic bypass. # [0] will allow all VLAN id's without any filtering. - # vlan-bpf-bypass: [] + # vlanBypass: [] # -- Clean all eBPF datapath state from the initContainer of the cilium-agent # DaemonSet. @@ -313,6 +328,12 @@ cleanBpfState: false # WARNING: Use with care! cleanState: false +# -- Wait for KUBE-PROXY-CANARY iptables rule to appear in "wait-for-kube-proxy" +# init container before launching cilium-agent. +# More context can be found in the commit message of below PR +# https://github.com/cilium/cilium/pull/20123 +waitForKubeProxy: false + cni: # -- Install the CNI configuration and binary files into the filesystem. install: true @@ -508,8 +529,14 @@ eni: # See https://github.com/aws/amazon-eks-pod-identity-webhook iamRole: "" # -- Filter via subnet IDs which will dictate which subnets are going to be used to create new ENIs + # Important note: This requires that each instance has an ENI with a matching subnet attached + # when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, + # use the CNI configuration file settings (cni.customConf) instead. subnetIDsFilter: "" # -- Filter via tags (k=v) which will dictate which subnets are going to be used to create new ENIs + # Important note: This requires that each instance has an ENI with a matching subnet attached + # when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, + # use the CNI configuration file settings (cni.customConf) instead. subnetTagsFilter: "" externalIPs: @@ -619,7 +646,19 @@ hubble: # Set this field ":4244" if you are enabling Hubble Relay, as it assumes that # Hubble is listening on port 4244. listenAddress: ":4244" - + peerService: + # -- Enable a K8s Service for the Peer service, so that it can be accessed + # by a non-local client + enabled: true + # -- Service Port for the Peer service. + # If not set, it is dynamically assigned to port 443 if TLS is enabled and to + # port 80 if not. + # servicePort: 80 + # -- Target Port for the Peer service. + targetPort: 4244 + # -- The cluster domain to use to query the Hubble Peer service. It should + # be the local cluster. + clusterDomain: cluster.local # -- TLS configuration for Hubble tls: # -- Enable mutual TLS for listenAddress. Setting this value to false is @@ -841,23 +880,10 @@ hubble: # memory: 64Mi # -- Resource requests and limits for the 'frontend' container of the 'hubble-ui' deployment. resources: {} - - proxy: - # -- Hubble-ui ingress proxy image. - image: - repository: quay.io/giantswarm/envoy - tag: v1.18.4 - pullPolicy: IfNotPresent - # [Example] - # resources: - # limits: - # cpu: 1000m - # memory: 1024M - # requests: - # cpu: 100m - # memory: 64Mi - # -- Resource requests and limits for the 'proxy' container of the 'hubble-ui' deployment. - resources: {} + server: + # -- Controls server listener for ipv6 + ipv6: + enabled: true # -- The number of replicas of Hubble UI to deploy. replicas: 1 @@ -1019,7 +1045,7 @@ l2NeighDiscovery: # -- Enable L2 neighbor discovery in the agent enabled: true # -- Override the agent's default neighbor resolution refresh period. - arping-refresh-period: "30s" + refreshPeriod: "30s" # -- Enable Layer 7 network policy. l7Proxy: true @@ -1339,6 +1365,10 @@ operator: # -- The priority class to use for cilium-operator priorityClassName: "" + # -- DNS policy for Cilium operator pods. + # Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy + dnsPolicy: "" + # -- cilium-operator update strategy updateStrategy: rollingUpdate: @@ -1426,6 +1456,9 @@ operator: # -- Interval for endpoint garbage collection. endpointGCInterval: "5m0s" + # -- Interval for cilium node garbage collection. + nodeGCInterval: "5m0s" + # -- Interval for identity garbage collection. identityGCInterval: "15m0s" @@ -1449,6 +1482,20 @@ operator: # -- Skip CRDs creation for cilium-operator skipCRDCreation: false + # -- Remove Cilium node taint from Kubernetes nodes that have a healthy Cilium + # pod running. + removeNodeTaints: true + + # -- Set Node condition NetworkUnavailable to 'false' with the reason + # 'CiliumIsUp' for nodes that have a healthy Cilium pod. + setNodeNetworkStatus: true + + unmanagedPodWatcher: + # -- Restart any pod that are not managed by Cilium. + restart: true + # -- Interval, in seconds, to check if there are any pods that are not + # managed by Cilium. + intervalSeconds: 15 nodeinit: # -- Enable the node initialization DaemonSet @@ -1526,7 +1573,7 @@ nodeinit: # -- bootstrapFile is the location of the file where the bootstrap timestamp is # written by the node-init DaemonSet - bootstrapFile: "/tmp/cilium-bootstrap-time" + bootstrapFile: "/tmp/cilium-bootstrap.d/cilium-bootstrap-time" preflight: # -- Enable Cilium pre-flight resources (required for upgrade) @@ -1638,6 +1685,33 @@ clustermesh: # -- Deploy clustermesh-apiserver for clustermesh useAPIServer: false + # -- Clustermesh explicit configuration. + config: + # -- Enable the Clustermesh explicit configuration. + enabled: false + # -- Default dns domain for the Clustermesh API servers + # This is used in the case cluster addresses are not provided + # and IPs are used. + domain: mesh.cilium.io + # -- List of clusters to be peered in the mesh. + clusters: [] + # clusters: + # # -- Name of the cluster + # - name: cluster1 + # # -- Address of the cluster, use this if you created DNS records for + # # the cluster Clustermesh API server. + # address: cluster1.mesh.cilium.io + # # -- Port of the cluster Clustermesh API server. + # port: 2379 + # # -- IPs of the cluster Clustermesh API server, use multiple ones when + # # you have multiple IPs to access the Clustermesh API server. + # ips: + # - 172.18.255.201 + # # -- base64 encoded PEM values for the cluster client certificate, private key and certificate authority. + # tls: + # cert: "" + # key: "" + apiserver: # -- Clustermesh API server image. image: @@ -1800,6 +1874,13 @@ cgroup: # in order to support graceful termination. enableK8sTerminatingEndpoint: true +# -- Configure whether to unload DNS policy rules on graceful shutdown +# dnsPolicyUnloadOnShutdown: false + +# -- Configure the key of the taint indicating that Cilium is not ready on the node. +# When set to a value starting with `ignore-taint.cluster-autoscaler.kubernetes.io/`, the Cluster Autoscaler will ignore the taint on its decisions, allowing the cluster to scale up. +agentNotReadyTaintKey: "node.cilium.io/agent-not-ready" + defaultPolicies: enabled: false # -- Node tolerations for default-policies job scheduling to nodes with taints