From 18821e6e7a3fec7ac1cd8dd3bc83942bf995a25e Mon Sep 17 00:00:00 2001 From: Harry Li Date: Tue, 3 Sep 2024 20:04:58 +0800 Subject: [PATCH 1/6] feat: coredns pod supports affinity and stain tolerance --- chart/templates/coredns-configmap.yaml | 8 +++++ chart/values.yaml | 48 ++++++++++++++------------ 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/chart/templates/coredns-configmap.yaml b/chart/templates/coredns-configmap.yaml index ee5575b0be..b39d546abf 100644 --- a/chart/templates/coredns-configmap.yaml +++ b/chart/templates/coredns-configmap.yaml @@ -159,6 +159,14 @@ data: {{- if .Values.controlPlane.coredns.deployment.nodeSelector }} {{ toYaml .Values.controlPlane.coredns.deployment.nodeSelector | indent 12 }} {{- end }} + {{- if .Values.controlPlane.coredns.deployment.affinity }} + affinity: +{{ toYaml .Values.controlPlane.coredns.deployment.affinity | indent 12 }} + {{- end }} + {{- if .Values.controlPlane.coredns.deployment.tolerations }} + tolerations: +{{ toYaml .Values.controlPlane.coredns.deployment.tolerations | indent 12 }} + {{- end }} {{- if .Values.controlPlane.coredns.deployment.topologySpreadConstraints }} topologySpreadConstraints: {{ toYaml .Values.controlPlane.coredns.deployment.topologySpreadConstraints | indent 12 }} diff --git a/chart/values.yaml b/chart/values.yaml index 5360444584..3bf57e667c 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -77,7 +77,7 @@ sync: # PersistentVolumes defines if persistent volumes created within the virtual cluster should get synced to the host cluster. persistentVolumes: enabled: false - + # Configure what resources vCluster should sync from the host cluster to the virtual cluster. fromHost: # Events defines if events should get synced from the host cluster to the virtual cluster, but not back. @@ -266,7 +266,7 @@ controlPlane: requests: cpu: 40m memory: 64Mi - + # BackingStore defines which backing store to use for virtual cluster. If not defined will use embedded database as a default backing store. backingStore: # Database defines that a database backend should be used as the backend for the virtual cluster. This uses a project called kine under the hood which is a shim for bridging Kubernetes and relational databases. @@ -387,7 +387,7 @@ controlPlane: headlessService: enabled: true annotations: {} - + # Proxy defines options for the virtual cluster control plane proxy that is used to do authentication and intercept requests. proxy: # BindAddress under which vCluster will expose the proxy. @@ -396,7 +396,7 @@ controlPlane: port: 8443 # ExtraSANs are extra hostnames to sign the vCluster proxy certificate for. extraSANs: [] - + # CoreDNS defines everything related to the coredns that is deployed and used within the vCluster. coredns: # Enabled defines if coredns is enabled @@ -430,6 +430,10 @@ controlPlane: annotations: {} # NodeSelector is the node selector to use for coredns. nodeSelector: {} + # Affinity is the affinity to apply to the pod. + affinity: {} + # Tolerations are the tolerations to apply to the pod. + tolerations: [] # Resources are the desired resources for coredns. resources: # Limits are resource limits for the container @@ -448,7 +452,7 @@ controlPlane: labelSelector: matchLabels: k8s-app: kube-dns - + # Service defines options for vCluster service deployed by Helm. service: # Enabled defines if the control plane service should be enabled @@ -462,7 +466,7 @@ controlPlane: # Spec allows you to configure extra service options. spec: type: ClusterIP - + # Ingress defines options for vCluster ingress deployed by Helm. ingress: # Enabled defines if the control plane ingress should be enabled @@ -479,7 +483,7 @@ controlPlane: # Spec allows you to configure extra ingress options. spec: tls: [] - + # StatefulSet defines options for vCluster statefulSet deployed by Helm. statefulSet: labels: {} @@ -595,14 +599,14 @@ controlPlane: # StartupProbe specifies if the startup probe for the container should be enabled startupProbe: enabled: true - + # ServiceMonitor can be used to automatically create a service monitor for vCluster deployment itself. serviceMonitor: # Enabled configures if Helm should create the service monitor. enabled: false labels: {} annotations: {} - + # Advanced holds additional configuration for the vCluster control plane. advanced: # DefaultImageRegistry will be used as a prefix for all internal images deployed by vCluster or Helm. This makes it easy to @@ -649,7 +653,7 @@ integrations: nodes: true # Pods defines if metrics-server pods api should get proxied from host to virtual cluster. pods: true - + # KubeVirt reuses a host kubevirt and makes certain CRDs from it available inside the vCluster kubeVirt: # Enabled signals if the integration should be enabled @@ -688,7 +692,7 @@ rbac: overwriteRules: [] # ExtraRules will add rules to the role. extraRules: [] - + # ClusterRole holds virtual cluster cluster role configuration clusterRole: # Enabled defines if the cluster role should be enabled or disabled. If auto, vCluster automatically determines whether the virtual cluster requires a cluster role. @@ -708,10 +712,10 @@ networking: toHost: [] # FromHost defines the services that should get synced from the host to the virtual cluster. fromHost: [] - + # ResolveDNS allows to define extra DNS rules. This only works if embedded coredns is configured. resolveDNS: [] - + # Advanced holds advanced network options. advanced: # ClusterDomain is the Kubernetes cluster domain to use within the virtual cluster. @@ -760,7 +764,7 @@ policies: matchExpressions: [] # Scopes are the resource quota scopes scopes: [] - + # LimitRange specifies limit range options. limitRange: # Enabled defines if the limit range should be deployed by vCluster. "auto" means that if resourceQuota is enabled, @@ -778,7 +782,7 @@ policies: ephemeral-storage: 3Gi memory: 128Mi cpu: 100m - + # NetworkPolicy specifies network policy options. networkPolicy: # Enabled defines if the network policy should be deployed by vCluster. @@ -806,7 +810,7 @@ policies: - 10.0.0.0/8 - 172.16.0.0/12 - 192.168.0.0/16 - + # CentralAdmission defines what validating or mutating webhooks should be enforced within the virtual cluster. centralAdmission: # ValidatingWebhooks are validating webhooks that should be enforced in the virtual cluster @@ -818,10 +822,10 @@ policies: exportKubeConfig: # Context is the name of the context within the generated kubeconfig to use. context: "" - + # Override the default https://localhost:8443 and specify a custom hostname for the generated kubeconfig. server: "" - + # Declare in which host cluster secret vCluster should store the generated virtual cluster kubeconfig. # If this is not defined, vCluster create it with `vc-NAME`. If you specify another name, # vCluster creates the config in this other secret. @@ -844,7 +848,7 @@ experimental: multiNamespaceMode: # Enabled specifies if multi namespace mode should get enabled enabled: false - + # SyncSettings are advanced settings for the syncer controller. syncSettings: # DisableSync will not sync any resources and disable most control plane functionality. @@ -855,12 +859,12 @@ experimental: targetNamespace: "" # SetOwner specifies if vCluster should set an owner reference on the synced objects to the vCluster service. This allows for easy garbage collection. setOwner: true - + # IsolatedControlPlane is a feature to run the vCluster control plane in a different Kubernetes cluster than the workloads themselves. isolatedControlPlane: # Headless states that Helm should deploy the vCluster in headless mode for the isolated control plane. headless: false - + # Deploy allows you to configure manifests and Helm charts to deploy within the virtual cluster. deploy: # Host defines what manifests to deploy into the host cluster @@ -877,7 +881,7 @@ experimental: manifestsTemplate: "" # Helm are Helm charts that should get deployed into the virtual cluster helm: [] - + # GenericSync holds options to generically sync resources from virtual cluster to host. genericSync: clusterRole: From 5fc15056ac150be71f7d4eed9270eb1277c0aa6d Mon Sep 17 00:00:00 2001 From: Harry Li Date: Wed, 4 Sep 2024 17:57:26 +0800 Subject: [PATCH 2/6] feat(config/config.go): CoreDNSDeployment struct add Affinity and Affinity filed --- config/config.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/config.go b/config/config.go index b5de050664..f0fb096beb 100644 --- a/config/config.go +++ b/config/config.go @@ -1193,6 +1193,12 @@ type CoreDNSDeployment struct { // NodeSelector is the node selector to use for coredns. NodeSelector map[string]string `json:"nodeSelector,omitempty"` + // Affinity is the affinity to apply to the pod. + Affinity map[string]interface{} `json:"affinity,omitempty"` + + // Tolerations are the tolerations to apply to the pod. + Tolerations []map[string]interface{} `json:"tolerations,omitempty"` + // Resources are the desired resources for coredns. Resources Resources `json:"resources,omitempty"` From 0689dc4323abf66e8c6d97a824b04eec70f46a27 Mon Sep 17 00:00:00 2001 From: Harry Li Date: Wed, 4 Sep 2024 22:24:30 +0800 Subject: [PATCH 3/6] feat(chart): regenerate values.schema.json --- chart/values.schema.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/chart/values.schema.json b/chart/values.schema.json index 22f01d2c94..6aad9717c5 100755 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -617,6 +617,17 @@ "type": "object", "description": "NodeSelector is the node selector to use for coredns." }, + "affinity": { + "type": "object", + "description": "Affinity is the affinity to apply to the pod." + }, + "tolerations": { + "items": { + "type": "object" + }, + "type": "array", + "description": "Tolerations are the tolerations to apply to the pod." + }, "resources": { "$ref": "#/$defs/Resources", "description": "Resources are the desired resources for coredns." From b9509ee637205783918c6c815745be7da46ee743 Mon Sep 17 00:00:00 2001 From: Harry Li Date: Wed, 4 Sep 2024 22:37:07 +0800 Subject: [PATCH 4/6] test (chart): add a test case to check coredns deployment affinity and tolerations --- chart/tests/coredns-configmap_test.yaml | 242 ++++++++++++++++++++++++ 1 file changed, 242 insertions(+) diff --git a/chart/tests/coredns-configmap_test.yaml b/chart/tests/coredns-configmap_test.yaml index a151087584..263ce2b8da 100644 --- a/chart/tests/coredns-configmap_test.yaml +++ b/chart/tests/coredns-configmap_test.yaml @@ -333,3 +333,245 @@ tests: - name: metrics port: 9153 protocol: TCP + - it: should correctly apply affinity and tolerations + set: + controlPlane: + coredns: + deployment: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - coredns + topologyKey: kubernetes.io/hostname + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + asserts: + - equal: + path: data["coredns.yaml"] + value: |- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: coredns + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns + subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns + namespace: kube-system + data: + Corefile: |- + .:1053 { + errors + health + ready + rewrite name regex .*\.nodes\.vcluster\.com kubernetes.default.svc.cluster.local + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + } + hosts /etc/NodeHosts { + ttl 60 + reload 15s + fallthrough + } + prometheus :9153 + forward . /etc/resolv.conf + cache 30 + loop + loadbalance + } + + import /etc/coredns/custom/*.server + NodeHosts: "" + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/name: "CoreDNS" + spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + spec: + priorityClassName: "" + serviceAccountName: coredns + nodeSelector: + kubernetes.io/os: linux + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - coredns + topologyKey: kubernetes.io/hostname + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + k8s-app: kube-dns + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + containers: + - name: coredns + image: {{.IMAGE}} + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 1000m + memory: 170Mi + requests: + cpu: 20m + memory: 64Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + - name: custom-config-volume + mountPath: /etc/coredns/custom + readOnly: true + securityContext: + runAsNonRoot: true + runAsUser: {{.RUN_AS_USER}} + runAsGroup: {{.RUN_AS_GROUP}} + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 2 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile + - key: NodeHosts + path: NodeHosts + - name: custom-config-volume + configMap: + name: coredns-custom + optional: true + --- + apiVersion: v1 + kind: Service + metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + spec: + type: ClusterIP + selector: + k8s-app: kube-dns + ports: + - name: dns + port: 53 + targetPort: 1053 + protocol: UDP + - name: dns-tcp + port: 53 + targetPort: 1053 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP From 658df9f44d91787d645f4fbd4ccab0bf68600f1c Mon Sep 17 00:00:00 2001 From: Harry Li Date: Wed, 4 Sep 2024 22:50:03 +0800 Subject: [PATCH 5/6] style(chart): delete unused spaces --- chart/values.yaml | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/chart/values.yaml b/chart/values.yaml index 3bf57e667c..085e685bc6 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -77,7 +77,7 @@ sync: # PersistentVolumes defines if persistent volumes created within the virtual cluster should get synced to the host cluster. persistentVolumes: enabled: false - + # Configure what resources vCluster should sync from the host cluster to the virtual cluster. fromHost: # Events defines if events should get synced from the host cluster to the virtual cluster, but not back. @@ -266,7 +266,7 @@ controlPlane: requests: cpu: 40m memory: 64Mi - + # BackingStore defines which backing store to use for virtual cluster. If not defined will use embedded database as a default backing store. backingStore: # Database defines that a database backend should be used as the backend for the virtual cluster. This uses a project called kine under the hood which is a shim for bridging Kubernetes and relational databases. @@ -387,7 +387,7 @@ controlPlane: headlessService: enabled: true annotations: {} - + # Proxy defines options for the virtual cluster control plane proxy that is used to do authentication and intercept requests. proxy: # BindAddress under which vCluster will expose the proxy. @@ -396,7 +396,7 @@ controlPlane: port: 8443 # ExtraSANs are extra hostnames to sign the vCluster proxy certificate for. extraSANs: [] - + # CoreDNS defines everything related to the coredns that is deployed and used within the vCluster. coredns: # Enabled defines if coredns is enabled @@ -452,7 +452,7 @@ controlPlane: labelSelector: matchLabels: k8s-app: kube-dns - + # Service defines options for vCluster service deployed by Helm. service: # Enabled defines if the control plane service should be enabled @@ -466,7 +466,7 @@ controlPlane: # Spec allows you to configure extra service options. spec: type: ClusterIP - + # Ingress defines options for vCluster ingress deployed by Helm. ingress: # Enabled defines if the control plane ingress should be enabled @@ -483,7 +483,7 @@ controlPlane: # Spec allows you to configure extra ingress options. spec: tls: [] - + # StatefulSet defines options for vCluster statefulSet deployed by Helm. statefulSet: labels: {} @@ -599,14 +599,14 @@ controlPlane: # StartupProbe specifies if the startup probe for the container should be enabled startupProbe: enabled: true - + # ServiceMonitor can be used to automatically create a service monitor for vCluster deployment itself. serviceMonitor: # Enabled configures if Helm should create the service monitor. enabled: false labels: {} annotations: {} - + # Advanced holds additional configuration for the vCluster control plane. advanced: # DefaultImageRegistry will be used as a prefix for all internal images deployed by vCluster or Helm. This makes it easy to @@ -653,7 +653,7 @@ integrations: nodes: true # Pods defines if metrics-server pods api should get proxied from host to virtual cluster. pods: true - + # KubeVirt reuses a host kubevirt and makes certain CRDs from it available inside the vCluster kubeVirt: # Enabled signals if the integration should be enabled @@ -692,7 +692,7 @@ rbac: overwriteRules: [] # ExtraRules will add rules to the role. extraRules: [] - + # ClusterRole holds virtual cluster cluster role configuration clusterRole: # Enabled defines if the cluster role should be enabled or disabled. If auto, vCluster automatically determines whether the virtual cluster requires a cluster role. @@ -712,10 +712,10 @@ networking: toHost: [] # FromHost defines the services that should get synced from the host to the virtual cluster. fromHost: [] - + # ResolveDNS allows to define extra DNS rules. This only works if embedded coredns is configured. resolveDNS: [] - + # Advanced holds advanced network options. advanced: # ClusterDomain is the Kubernetes cluster domain to use within the virtual cluster. @@ -764,7 +764,7 @@ policies: matchExpressions: [] # Scopes are the resource quota scopes scopes: [] - + # LimitRange specifies limit range options. limitRange: # Enabled defines if the limit range should be deployed by vCluster. "auto" means that if resourceQuota is enabled, @@ -782,7 +782,7 @@ policies: ephemeral-storage: 3Gi memory: 128Mi cpu: 100m - + # NetworkPolicy specifies network policy options. networkPolicy: # Enabled defines if the network policy should be deployed by vCluster. @@ -810,7 +810,7 @@ policies: - 10.0.0.0/8 - 172.16.0.0/12 - 192.168.0.0/16 - + # CentralAdmission defines what validating or mutating webhooks should be enforced within the virtual cluster. centralAdmission: # ValidatingWebhooks are validating webhooks that should be enforced in the virtual cluster @@ -822,10 +822,10 @@ policies: exportKubeConfig: # Context is the name of the context within the generated kubeconfig to use. context: "" - + # Override the default https://localhost:8443 and specify a custom hostname for the generated kubeconfig. server: "" - + # Declare in which host cluster secret vCluster should store the generated virtual cluster kubeconfig. # If this is not defined, vCluster create it with `vc-NAME`. If you specify another name, # vCluster creates the config in this other secret. @@ -848,7 +848,7 @@ experimental: multiNamespaceMode: # Enabled specifies if multi namespace mode should get enabled enabled: false - + # SyncSettings are advanced settings for the syncer controller. syncSettings: # DisableSync will not sync any resources and disable most control plane functionality. @@ -859,12 +859,12 @@ experimental: targetNamespace: "" # SetOwner specifies if vCluster should set an owner reference on the synced objects to the vCluster service. This allows for easy garbage collection. setOwner: true - + # IsolatedControlPlane is a feature to run the vCluster control plane in a different Kubernetes cluster than the workloads themselves. isolatedControlPlane: # Headless states that Helm should deploy the vCluster in headless mode for the isolated control plane. headless: false - + # Deploy allows you to configure manifests and Helm charts to deploy within the virtual cluster. deploy: # Host defines what manifests to deploy into the host cluster @@ -881,7 +881,7 @@ experimental: manifestsTemplate: "" # Helm are Helm charts that should get deployed into the virtual cluster helm: [] - + # GenericSync holds options to generically sync resources from virtual cluster to host. genericSync: clusterRole: From 16f462fdb4e898fc8029398e7a6268f925b58e96 Mon Sep 17 00:00:00 2001 From: Harry Li Date: Wed, 11 Sep 2024 11:52:51 +0800 Subject: [PATCH 6/6] feat(config/values.yaml): coredns add affinity and tolerations --- config/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/values.yaml b/config/values.yaml index 1d6161cb3b..9fd96940b3 100644 --- a/config/values.yaml +++ b/config/values.yaml @@ -240,6 +240,8 @@ controlPlane: labels: {} annotations: {} nodeSelector: {} + affinity: {} + tolerations: [] resources: limits: cpu: 1000m