From f754cbda7d81df2848de017754587ba9acb4a9fe Mon Sep 17 00:00:00 2001 From: Salvatore Mazzarino Date: Mon, 30 Mar 2020 15:23:19 +0200 Subject: [PATCH 1/5] NodeLocal DNSCache Signed-off-by: Salvatore Mazzarino --- docs/cluster_spec.md | 10 + k8s/crds/kops.k8s.io_clusters.yaml | 23 +++ pkg/apis/kops/cluster.go | 16 ++ pkg/apis/kops/v1alpha2/cluster.go | 16 ++ .../kops/v1alpha2/zz_generated.conversion.go | 56 ++++++ .../kops/v1alpha2/zz_generated.deepcopy.go | 21 ++ pkg/apis/kops/validation/legacy.go | 30 +++ pkg/apis/kops/zz_generated.deepcopy.go | 21 ++ pkg/model/components/kubedns.go | 25 +++ .../k8s-1.12.yaml.template | 182 ++++++++++++++++++ .../pkg/fi/cloudup/bootstrapchannelbuilder.go | 23 +++ 11 files changed, 423 insertions(+) create mode 100644 upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index 0ab8981d292ee..a2563fe355909 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -543,6 +543,16 @@ spec: **Note:** If you are upgrading to CoreDNS, kube-dns will be left in place and must be removed manually (you can scale the kube-dns and kube-dns-autoscaler deployments in the `kube-system` namespace to 0 as a starting point). The `kube-dns` Service itself should be left in place, as this retains the ClusterIP and eliminates the possibility of DNS outages in your cluster. If you would like to continue autoscaling, update the `kube-dns-autoscaler` Deployment container command for `--target=Deployment/kube-dns` to be `--target=Deployment/coredns`. +If you are using CoreDNS, you can enable NodeLocal DNSCache. It is used to improve improve the Cluster DNS performance by running a dns caching agent on cluster nodes as a DaemonSet. + +```yaml +spec: + kubeDNS: + provider: CoreDNS + nodeLocalDNS: + enabled: true +``` + ### kubeControllerManager This block contains configurations for the `controller-manager`. diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 64c197e1082d9..1a32aa887d1eb 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -1611,6 +1611,29 @@ spec: dns container in the cluster. Default 70m. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + nodeLocalDNS: + description: NodeLocalDNS specifies the configuration for the + node-local-dns addon + properties: + clusterIP: + description: ClusterIP is the cluster ip + type: string + domain: + description: Domain is the dns domain + type: string + enabled: + description: Disable indicates we do not wish to run the node-local-dns + addon + type: boolean + localIP: + description: Local listen IP address. It can be any IP in + the 169.254.20.0/16 space or any other IP address that can + be guaranteed to not collide with any existing IP. + type: string + serverIP: + description: ServerIP is the server ip + type: string + type: object provider: description: Provider indicates whether CoreDNS or kube-dns will be the default service discovery. diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 5896b2c86fb45..ccbe4868a8e74 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -400,6 +400,22 @@ type KubeDNSConfig struct { CPURequest *resource.Quantity `json:"cpuRequest,omitempty"` // MemoryLimit specifies the memory limit of each dns container in the cluster. Default 170m. MemoryLimit *resource.Quantity `json:"memoryLimit,omitempty"` + // NodeLocalDNS specifies the configuration for the node-local-dns addon + NodeLocalDNS *NodeLocalDNSConfig `json:"nodeLocalDNS,omitempty"` +} + +// NodeLocalDNSConfig are options of the node-local-dns +type NodeLocalDNSConfig struct { + // Disable indicates we do not wish to run the node-local-dns addon + Enabled bool `json:"enabled,omitempty"` + // Domain is the dns domain + Domain string `json:"domain,omitempty"` + // Local listen IP address. It can be any IP in the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP. + LocalIP string `json:"localIP,omitempty"` + // ServerIP is the server ip + ServerIP string `json:"serverIP,omitempty"` + // ClusterIP is the cluster ip + ClusterIP string `json:"clusterIP,omitempty"` } // ExternalDNSConfig are options of the dns-controller diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index af05ba7d7178a..25989ece353bc 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -398,6 +398,22 @@ type KubeDNSConfig struct { CPURequest *resource.Quantity `json:"cpuRequest,omitempty"` // MemoryLimit specifies the memory limit of each dns container in the cluster. Default 170m. MemoryLimit *resource.Quantity `json:"memoryLimit,omitempty"` + // NodeLocalDNS specifies the configuration for the node-local-dns addon + NodeLocalDNS *NodeLocalDNSConfig `json:"nodeLocalDNS,omitempty"` +} + +// NodeLocalDNSConfig are options of the node-local-dns +type NodeLocalDNSConfig struct { + // Disable indicates we do not wish to run the node-local-dns addon + Enabled bool `json:"enabled,omitempty"` + // Domain is the dns domain + Domain string `json:"domain,omitempty"` + // Local listen IP address. It can be any IP in the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP. + LocalIP string `json:"localIP,omitempty"` + // ServerIP is the server ip + ServerIP string `json:"serverIP,omitempty"` + // ClusterIP is the cluster ip + ClusterIP string `json:"clusterIP,omitempty"` } // ExternalDNSConfig are options of the dns-controller diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index f55708fc474b2..5f9b04ba3b349 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -703,6 +703,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*NodeLocalDNSConfig)(nil), (*kops.NodeLocalDNSConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(a.(*NodeLocalDNSConfig), b.(*kops.NodeLocalDNSConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.NodeLocalDNSConfig)(nil), (*NodeLocalDNSConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(a.(*kops.NodeLocalDNSConfig), b.(*NodeLocalDNSConfig), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*OpenstackBlockStorageConfig)(nil), (*kops.OpenstackBlockStorageConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(a.(*OpenstackBlockStorageConfig), b.(*kops.OpenstackBlockStorageConfig), scope) }); err != nil { @@ -3965,6 +3975,15 @@ func autoConvert_v1alpha2_KubeDNSConfig_To_kops_KubeDNSConfig(in *KubeDNSConfig, out.MemoryRequest = in.MemoryRequest out.CPURequest = in.CPURequest out.MemoryLimit = in.MemoryLimit + if in.NodeLocalDNS != nil { + in, out := &in.NodeLocalDNS, &out.NodeLocalDNS + *out = new(kops.NodeLocalDNSConfig) + if err := Convert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(*in, *out, s); err != nil { + return err + } + } else { + out.NodeLocalDNS = nil + } return nil } @@ -3988,6 +4007,15 @@ func autoConvert_kops_KubeDNSConfig_To_v1alpha2_KubeDNSConfig(in *kops.KubeDNSCo out.MemoryRequest = in.MemoryRequest out.CPURequest = in.CPURequest out.MemoryLimit = in.MemoryLimit + if in.NodeLocalDNS != nil { + in, out := &in.NodeLocalDNS, &out.NodeLocalDNS + *out = new(NodeLocalDNSConfig) + if err := Convert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(*in, *out, s); err != nil { + return err + } + } else { + out.NodeLocalDNS = nil + } return nil } @@ -4808,6 +4836,34 @@ func Convert_kops_NodeAuthorizerSpec_To_v1alpha2_NodeAuthorizerSpec(in *kops.Nod return autoConvert_kops_NodeAuthorizerSpec_To_v1alpha2_NodeAuthorizerSpec(in, out, s) } +func autoConvert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLocalDNSConfig, out *kops.NodeLocalDNSConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + out.Domain = in.Domain + out.LocalIP = in.LocalIP + out.ServerIP = in.ServerIP + out.ClusterIP = in.ClusterIP + return nil +} + +// Convert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig is an autogenerated conversion function. +func Convert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLocalDNSConfig, out *kops.NodeLocalDNSConfig, s conversion.Scope) error { + return autoConvert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in, out, s) +} + +func autoConvert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(in *kops.NodeLocalDNSConfig, out *NodeLocalDNSConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + out.Domain = in.Domain + out.LocalIP = in.LocalIP + out.ServerIP = in.ServerIP + out.ClusterIP = in.ClusterIP + return nil +} + +// Convert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig is an autogenerated conversion function. +func Convert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(in *kops.NodeLocalDNSConfig, out *NodeLocalDNSConfig, s conversion.Scope) error { + return autoConvert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(in, out, s) +} + func autoConvert_v1alpha2_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(in *OpenstackBlockStorageConfig, out *kops.OpenstackBlockStorageConfig, s conversion.Scope) error { out.Version = in.Version out.IgnoreAZ = in.IgnoreAZ diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index e58ab0775a55b..d893ebf068dd2 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -2451,6 +2451,11 @@ func (in *KubeDNSConfig) DeepCopyInto(out *KubeDNSConfig) { x := (*in).DeepCopy() *out = &x } + if in.NodeLocalDNS != nil { + in, out := &in.NodeLocalDNS, &out.NodeLocalDNS + *out = new(NodeLocalDNSConfig) + **out = **in + } return } @@ -3175,6 +3180,22 @@ func (in *NodeAuthorizerSpec) DeepCopy() *NodeAuthorizerSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeLocalDNSConfig) DeepCopyInto(out *NodeLocalDNSConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeLocalDNSConfig. +func (in *NodeLocalDNSConfig) DeepCopy() *NodeLocalDNSConfig { + if in == nil { + return nil + } + out := new(NodeLocalDNSConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenstackBlockStorageConfig) DeepCopyInto(out *OpenstackBlockStorageConfig) { *out = *in diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go index 69fc536629b54..d9d199ae56e9c 100644 --- a/pkg/apis/kops/validation/legacy.go +++ b/pkg/apis/kops/validation/legacy.go @@ -285,6 +285,36 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList { } } } + + // @ check that NodeLocalDNS addon is configured correctly + if c.Spec.KubeDNS.NodeLocalDNS != nil && c.Spec.KubeDNS.NodeLocalDNS.Enabled { + if c.Spec.KubeDNS.Provider != "CoreDNS" { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "provider"), "KubeDNS provider must be set to CoreDNS if NodeLocalDNS addon is enabled")) + } + + if c.Spec.KubeDNS.NodeLocalDNS.LocalIP == "" { + if c.Spec.Kubelet != nil && c.Spec.Kubelet.ClusterDNS != c.Spec.KubeDNS.NodeLocalDNS.LocalIP { + if c.Spec.KubeProxy != nil && c.Spec.KubeProxy.ProxyMode == "ipvs" { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubelet", "clusterDNS"), "Kubelet ClusterDNS must be set to the default IP address for LocalIP when KubeProxy ProxyMode is set to ipvs")) + } + + // in case that the CNI is set to Cilium, it is necessary that ClusterDNS points to the NodeLocal DNSCache local IP addr + if c.Spec.Networking != nil && c.Spec.Networking.Cilium != nil { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubelet", "clusterDNS"), "Kubelet ClusterDNS must be set to the default IP address for LocalIP when CNI is Cilium")) + } + } + } else { + address := c.Spec.KubeDNS.NodeLocalDNS.LocalIP + ip := net.ParseIP(address) + if ip == nil { + allErrs = append(allErrs, field.Invalid(fieldSpec.Child("kubeDNS", "nodeLocalDNS", "localIP"), address, "Cluster had an invalid kubeDNS.nodeLocalDNS.localIP")) + } else { + if c.Spec.Kubelet != nil && c.Spec.Kubelet.ClusterDNS != c.Spec.KubeDNS.NodeLocalDNS.LocalIP { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "nodeLocalDNS", "localIP"), fmt.Sprintf("kubelet.ClusterDNS had an IP address %q that did not match %q", c.Spec.Kubelet.ClusterDNS, c.Spec.KubeDNS.NodeLocalDNS.LocalIP))) + } + } + } + } } // @check the nameservers are valid diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 5f00976e9e65b..e169e05c1502b 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -2633,6 +2633,11 @@ func (in *KubeDNSConfig) DeepCopyInto(out *KubeDNSConfig) { x := (*in).DeepCopy() *out = &x } + if in.NodeLocalDNS != nil { + in, out := &in.NodeLocalDNS, &out.NodeLocalDNS + *out = new(NodeLocalDNSConfig) + **out = **in + } return } @@ -3373,6 +3378,22 @@ func (in *NodeAuthorizerSpec) DeepCopy() *NodeAuthorizerSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeLocalDNSConfig) DeepCopyInto(out *NodeLocalDNSConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeLocalDNSConfig. +func (in *NodeLocalDNSConfig) DeepCopy() *NodeLocalDNSConfig { + if in == nil { + return nil + } + out := new(NodeLocalDNSConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NoopStatusStore) DeepCopyInto(out *NoopStatusStore) { *out = *in diff --git a/pkg/model/components/kubedns.go b/pkg/model/components/kubedns.go index 3c8ba3ce343b1..0a4077bae206e 100644 --- a/pkg/model/components/kubedns.go +++ b/pkg/model/components/kubedns.go @@ -74,5 +74,30 @@ func (b *KubeDnsOptionsBuilder) BuildOptions(o interface{}) error { clusterSpec.KubeDNS.MemoryLimit = &defaultMemoryLimit } + NodeLocalDNS := clusterSpec.KubeDNS.NodeLocalDNS + if NodeLocalDNS == nil { + NodeLocalDNS = &kops.NodeLocalDNSConfig{} + NodeLocalDNS.Enabled = false + } else if NodeLocalDNS.Enabled { + // https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/#configuration + NodeLocalDNS.Domain = clusterSpec.ClusterDNSDomain + + switch clusterSpec.KubeProxy.ProxyMode { + case "iptables": + NodeLocalDNS.ServerIP = clusterSpec.KubeDNS.ServerIP + // This will be pushed into the Corefile and replaced by NodeLocal DNSCache at startup + NodeLocalDNS.ClusterIP = "__PILLAR__CLUSTER__DNS__" + + case "ipvs": + NodeLocalDNS.ServerIP = "" + NodeLocalDNS.ClusterIP = clusterSpec.KubeDNS.ServerIP + + default: + // the default supposes the kube-proxy working in iptables mode + NodeLocalDNS.ServerIP = clusterSpec.KubeDNS.ServerIP + NodeLocalDNS.ClusterIP = "__PILLAR__CLUSTER__DNS__" + } + } + return nil } diff --git a/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template new file mode 100644 index 0000000000000..4797317d9a11c --- /dev/null +++ b/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template @@ -0,0 +1,182 @@ +# Vendored from https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-local-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns-upstream + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeDNSUpstream" +spec: + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-local-dns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + Corefile: | + {{ .KubeDNS.NodeLocalDNS.Domain }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }} + forward . {{ .KubeDNS.NodeLocalDNS.ClusterIP }} { + force_tcp + } + prometheus :9253 + health {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }}:8080 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }} + forward . {{ .KubeDNS.NodeLocalDNS.ClusterIP }} { + force_tcp + } + prometheus :9253 + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }} + forward . {{ .KubeDNS.NodeLocalDNS.ClusterIP }} { + force_tcp + } + prometheus :9253 + } + .:53 { + errors + cache 30 + reload + loop + bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }} + forward . __PILLAR__UPSTREAM__SERVERS__ { + force_tcp + } + prometheus :9253 + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-local-dns + namespace: kube-system + labels: + k8s-app: node-local-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + k8s-app: node-local-dns + template: + metadata: + labels: + k8s-app: node-local-dns + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: node-local-dns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + containers: + - name: node-cache + image: k8s.gcr.io/k8s-dns-node-cache:1.15.10 + resources: + requests: + cpu: 25m + memory: 5Mi + {{ if .KubeDNS.NodeLocalDNS.ServerIP }} + args: [ "-localip", "{{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }},{{ .KubeDNS.NodeLocalDNS.ServerIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + {{ else }} + args: [ "-localip", "{{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + {{ end }} + securityContext: + privileged: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + host: {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} + path: /health + port: 8080 + initialDelaySeconds: 60 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - name: config-volume + mountPath: /etc/coredns + - name: kube-dns-config + mountPath: /etc/kube-dns + volumes: + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + - name: config-volume + configMap: + name: node-local-dns + items: + - key: Corefile + path: Corefile.base \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index e47f6ca1541b3..df2a40b9e44b6 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -451,6 +451,29 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { } } + // @check the node-local-dns has not been disabled + NodeLocalDNS := b.cluster.Spec.KubeDNS.NodeLocalDNS + if kubeDNS.Provider == "CoreDNS" && NodeLocalDNS != nil && NodeLocalDNS.Enabled { + { + key := "nodelocaldns.addons.k8s.io" + version := "1.18.0" + + { + location := key + "/k8s-1.12.yaml" + id := "k8s-1.12" + + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(version), + Selector: map[string]string{"k8s-addon": key}, + Manifest: fi.String(location), + KubernetesVersion: ">=1.12.0", + Id: id, + }) + } + } + } + if kops.CloudProviderID(b.cluster.Spec.CloudProvider) == kops.CloudProviderAWS { key := "storage-aws.addons.k8s.io" version := "1.15.0" From cf57350e4c94ba8c0f9419321cda325652053938 Mon Sep 17 00:00:00 2001 From: Salvatore Mazzarino Date: Mon, 30 Mar 2020 16:53:36 +0200 Subject: [PATCH 2/5] Improved validation logic Signed-off-by: Salvatore Mazzarino --- pkg/apis/kops/validation/legacy.go | 23 +---------------------- pkg/apis/kops/validation/validation.go | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go index d9d199ae56e9c..4658370699442 100644 --- a/pkg/apis/kops/validation/legacy.go +++ b/pkg/apis/kops/validation/legacy.go @@ -292,28 +292,7 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList { allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "provider"), "KubeDNS provider must be set to CoreDNS if NodeLocalDNS addon is enabled")) } - if c.Spec.KubeDNS.NodeLocalDNS.LocalIP == "" { - if c.Spec.Kubelet != nil && c.Spec.Kubelet.ClusterDNS != c.Spec.KubeDNS.NodeLocalDNS.LocalIP { - if c.Spec.KubeProxy != nil && c.Spec.KubeProxy.ProxyMode == "ipvs" { - allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubelet", "clusterDNS"), "Kubelet ClusterDNS must be set to the default IP address for LocalIP when KubeProxy ProxyMode is set to ipvs")) - } - - // in case that the CNI is set to Cilium, it is necessary that ClusterDNS points to the NodeLocal DNSCache local IP addr - if c.Spec.Networking != nil && c.Spec.Networking.Cilium != nil { - allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubelet", "clusterDNS"), "Kubelet ClusterDNS must be set to the default IP address for LocalIP when CNI is Cilium")) - } - } - } else { - address := c.Spec.KubeDNS.NodeLocalDNS.LocalIP - ip := net.ParseIP(address) - if ip == nil { - allErrs = append(allErrs, field.Invalid(fieldSpec.Child("kubeDNS", "nodeLocalDNS", "localIP"), address, "Cluster had an invalid kubeDNS.nodeLocalDNS.localIP")) - } else { - if c.Spec.Kubelet != nil && c.Spec.Kubelet.ClusterDNS != c.Spec.KubeDNS.NodeLocalDNS.LocalIP { - allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "nodeLocalDNS", "localIP"), fmt.Sprintf("kubelet.ClusterDNS had an IP address %q that did not match %q", c.Spec.Kubelet.ClusterDNS, c.Spec.KubeDNS.NodeLocalDNS.LocalIP))) - } - } - } + allErrs = append(allErrs, validateNodeLocalDNS(&c.Spec, fieldSpec.Child("spec"))...) } } diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index d30e5d72d108a..77ad44b9b2894 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -646,3 +646,27 @@ func validateRollingUpdate(rollingUpdate *kops.RollingUpdate, fldpath *field.Pat return allErrs } + +func validateNodeLocalDNS(spec *kops.ClusterSpec, fldpath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if spec.KubeDNS.NodeLocalDNS.LocalIP != "" { + address := spec.KubeDNS.NodeLocalDNS.LocalIP + ip := net.ParseIP(address) + if ip == nil { + allErrs = append(allErrs, field.Invalid(fldpath.Child("kubeDNS", "nodeLocalDNS", "localIP"), address, "Cluster had an invalid kubeDNS.nodeLocalDNS.localIP")) + } + } + + if (spec.KubeProxy != nil && spec.KubeProxy.ProxyMode == "ipvs") || (spec.Networking != nil && spec.Networking.Cilium != nil) { + if spec.Kubelet != nil && spec.Kubelet.ClusterDNS != spec.KubeDNS.NodeLocalDNS.LocalIP { + allErrs = append(allErrs, field.Forbidden(fldpath.Child("kubelet", "clusterDNS"), "Kubelet ClusterDNS must be set to the default IP address for LocalIP")) + } + + if spec.MasterKubelet != nil && spec.MasterKubelet.ClusterDNS != spec.KubeDNS.NodeLocalDNS.LocalIP { + allErrs = append(allErrs, field.Forbidden(fldpath.Child("kubelet", "clusterDNS"), "MasterKubelet ClusterDNS must be set to the default IP address for LocalIP")) + } + } + + return allErrs +} From d18c88a546ba0659acaf6f2274d9e772f7ed7e1d Mon Sep 17 00:00:00 2001 From: Salvatore Mazzarino Date: Mon, 30 Mar 2020 17:35:24 +0200 Subject: [PATCH 3/5] Add unit tests Signed-off-by: Salvatore Mazzarino --- pkg/apis/kops/validation/validation_test.go | 85 +++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/pkg/apis/kops/validation/validation_test.go b/pkg/apis/kops/validation/validation_test.go index 037b6bce9ff1d..52ca54d6e2214 100644 --- a/pkg/apis/kops/validation/validation_test.go +++ b/pkg/apis/kops/validation/validation_test.go @@ -534,3 +534,88 @@ func Test_Validate_RollingUpdate(t *testing.T) { func intStr(i intstr.IntOrString) *intstr.IntOrString { return &i } + +func Test_Validate_NodeLocalDNS(t *testing.T) { + grid := []struct { + Input kops.ClusterSpec + ExpectedErrors []string + }{ + { + Input: kops.ClusterSpec{ + KubeProxy: &kops.KubeProxyConfig{ + ProxyMode: "iptables", + }, + KubeDNS: &kops.KubeDNSConfig{ + Provider: "CoreDNS", + NodeLocalDNS: &kops.NodeLocalDNSConfig{ + Enabled: true, + }, + }, + }, + ExpectedErrors: []string{}, + }, + { + Input: kops.ClusterSpec{ + Kubelet: &kops.KubeletConfigSpec{ + ClusterDNS: "100.64.0.10", + }, + KubeProxy: &kops.KubeProxyConfig{ + ProxyMode: "ipvs", + }, + KubeDNS: &kops.KubeDNSConfig{ + Provider: "CoreDNS", + NodeLocalDNS: &kops.NodeLocalDNSConfig{ + Enabled: true, + }, + }, + }, + ExpectedErrors: []string{"Forbidden::spec.kubelet.clusterDNS"}, + }, + { + Input: kops.ClusterSpec{ + Kubelet: &kops.KubeletConfigSpec{ + ClusterDNS: "100.64.0.10", + }, + KubeProxy: &kops.KubeProxyConfig{ + ProxyMode: "ipvs", + }, + KubeDNS: &kops.KubeDNSConfig{ + Provider: "CoreDNS", + NodeLocalDNS: &kops.NodeLocalDNSConfig{ + Enabled: true, + }, + }, + Networking: &kops.NetworkingSpec{ + Cilium: &kops.CiliumNetworkingSpec{}, + }, + }, + ExpectedErrors: []string{"Forbidden::spec.kubelet.clusterDNS"}, + }, + { + Input: kops.ClusterSpec{ + Kubelet: &kops.KubeletConfigSpec{ + ClusterDNS: "169.254.20.10", + }, + KubeProxy: &kops.KubeProxyConfig{ + ProxyMode: "iptables", + }, + KubeDNS: &kops.KubeDNSConfig{ + Provider: "CoreDNS", + NodeLocalDNS: &kops.NodeLocalDNSConfig{ + Enabled: true, + LocalIP: "169.254.20.10", + }, + }, + Networking: &kops.NetworkingSpec{ + Cilium: &kops.CiliumNetworkingSpec{}, + }, + }, + ExpectedErrors: []string{}, + }, + } + + for _, g := range grid { + errs := validateNodeLocalDNS(&g.Input, field.NewPath("spec")) + testErrors(t, g.Input, errs, g.ExpectedErrors) + } +} From d5019a6c115895affecbc9565d21358466edd3ae Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Tue, 31 Mar 2020 14:42:40 +0200 Subject: [PATCH 4/5] Simplify the spec and templates a bit --- docs/cluster_spec.md | 10 +++++++ k8s/crds/kops.k8s.io_clusters.yaml | 9 ------- pkg/apis/kops/cluster.go | 6 ----- pkg/apis/kops/v1alpha2/cluster.go | 6 ----- .../kops/v1alpha2/zz_generated.conversion.go | 6 ----- pkg/apis/kops/validation/legacy.go | 14 +++++++--- pkg/model/components/kubedns.go | 21 ++------------- .../k8s-1.12.yaml.template | 26 +++++++++---------- upup/pkg/fi/cloudup/template_functions.go | 15 +++++++++++ 9 files changed, 50 insertions(+), 63 deletions(-) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index a2563fe355909..eb23177e08633 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -553,6 +553,16 @@ spec: enabled: true ``` +If you are using kube-proxy in ipvs mode or Cilium as CNI, you have to set the nodeLocalDNS as ClusterDNS. + +```yaml +spec: + kubelet: + clusterDNS: 169.254.20.10 + masterKubelet: + clusterDNS: 169.254.20.10 +``` + ### kubeControllerManager This block contains configurations for the `controller-manager`. diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 1a32aa887d1eb..42a48128fdb13 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -1615,12 +1615,6 @@ spec: description: NodeLocalDNS specifies the configuration for the node-local-dns addon properties: - clusterIP: - description: ClusterIP is the cluster ip - type: string - domain: - description: Domain is the dns domain - type: string enabled: description: Disable indicates we do not wish to run the node-local-dns addon @@ -1630,9 +1624,6 @@ spec: the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP. type: string - serverIP: - description: ServerIP is the server ip - type: string type: object provider: description: Provider indicates whether CoreDNS or kube-dns will diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index ccbe4868a8e74..ba3c1c407f8ad 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -408,14 +408,8 @@ type KubeDNSConfig struct { type NodeLocalDNSConfig struct { // Disable indicates we do not wish to run the node-local-dns addon Enabled bool `json:"enabled,omitempty"` - // Domain is the dns domain - Domain string `json:"domain,omitempty"` // Local listen IP address. It can be any IP in the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP. LocalIP string `json:"localIP,omitempty"` - // ServerIP is the server ip - ServerIP string `json:"serverIP,omitempty"` - // ClusterIP is the cluster ip - ClusterIP string `json:"clusterIP,omitempty"` } // ExternalDNSConfig are options of the dns-controller diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 25989ece353bc..53ec1a1069a02 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -406,14 +406,8 @@ type KubeDNSConfig struct { type NodeLocalDNSConfig struct { // Disable indicates we do not wish to run the node-local-dns addon Enabled bool `json:"enabled,omitempty"` - // Domain is the dns domain - Domain string `json:"domain,omitempty"` // Local listen IP address. It can be any IP in the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP. LocalIP string `json:"localIP,omitempty"` - // ServerIP is the server ip - ServerIP string `json:"serverIP,omitempty"` - // ClusterIP is the cluster ip - ClusterIP string `json:"clusterIP,omitempty"` } // ExternalDNSConfig are options of the dns-controller diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 5f9b04ba3b349..d6142016455a9 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -4838,10 +4838,7 @@ func Convert_kops_NodeAuthorizerSpec_To_v1alpha2_NodeAuthorizerSpec(in *kops.Nod func autoConvert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLocalDNSConfig, out *kops.NodeLocalDNSConfig, s conversion.Scope) error { out.Enabled = in.Enabled - out.Domain = in.Domain out.LocalIP = in.LocalIP - out.ServerIP = in.ServerIP - out.ClusterIP = in.ClusterIP return nil } @@ -4852,10 +4849,7 @@ func Convert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLoca func autoConvert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(in *kops.NodeLocalDNSConfig, out *NodeLocalDNSConfig, s conversion.Scope) error { out.Enabled = in.Enabled - out.Domain = in.Domain out.LocalIP = in.LocalIP - out.ServerIP = in.ServerIP - out.ClusterIP = in.ClusterIP return nil } diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go index 4658370699442..22c68fbbcff8d 100644 --- a/pkg/apis/kops/validation/legacy.go +++ b/pkg/apis/kops/validation/legacy.go @@ -277,11 +277,11 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList { allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "serverIP"), fmt.Sprintf("ServiceClusterIPRange %q must contain the DNS Server IP %q", c.Spec.ServiceClusterIPRange, address))) } if !featureflag.ExperimentalClusterDNS.Enabled() { - if c.Spec.Kubelet != nil && c.Spec.Kubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP { - allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "serverIP"), "Kubelet ClusterDNS did not match cluster kubeDNS.serverIP")) + if isExperimentalClusterDNS(c.Spec.Kubelet, c.Spec.KubeDNS) { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubelet", "clusterDNS"), "Kubelet ClusterDNS did not match cluster kubeDNS.serverIP or nodeLocalDNS.localIP")) } - if c.Spec.MasterKubelet != nil && c.Spec.MasterKubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP { - allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "serverIP"), "MasterKubelet ClusterDNS did not match cluster kubeDNS.serverIP")) + if isExperimentalClusterDNS(c.Spec.MasterKubelet, c.Spec.KubeDNS) { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("masterKubelet", "clusterDNS"), "MasterKubelet ClusterDNS did not match cluster kubeDNS.serverIP or nodeLocalDNS.localIP")) } } } @@ -714,3 +714,9 @@ func validateKubelet(k *kops.KubeletConfigSpec, c *kops.Cluster, kubeletPath *fi } return allErrs } + +func isExperimentalClusterDNS(k *kops.KubeletConfigSpec, dns *kops.KubeDNSConfig) bool { + + return k != nil && k.ClusterDNS != dns.ServerIP && dns.NodeLocalDNS != nil && k.ClusterDNS != dns.NodeLocalDNS.LocalIP + +} diff --git a/pkg/model/components/kubedns.go b/pkg/model/components/kubedns.go index 0a4077bae206e..3e56fdad8ac96 100644 --- a/pkg/model/components/kubedns.go +++ b/pkg/model/components/kubedns.go @@ -78,25 +78,8 @@ func (b *KubeDnsOptionsBuilder) BuildOptions(o interface{}) error { if NodeLocalDNS == nil { NodeLocalDNS = &kops.NodeLocalDNSConfig{} NodeLocalDNS.Enabled = false - } else if NodeLocalDNS.Enabled { - // https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/#configuration - NodeLocalDNS.Domain = clusterSpec.ClusterDNSDomain - - switch clusterSpec.KubeProxy.ProxyMode { - case "iptables": - NodeLocalDNS.ServerIP = clusterSpec.KubeDNS.ServerIP - // This will be pushed into the Corefile and replaced by NodeLocal DNSCache at startup - NodeLocalDNS.ClusterIP = "__PILLAR__CLUSTER__DNS__" - - case "ipvs": - NodeLocalDNS.ServerIP = "" - NodeLocalDNS.ClusterIP = clusterSpec.KubeDNS.ServerIP - - default: - // the default supposes the kube-proxy working in iptables mode - NodeLocalDNS.ServerIP = clusterSpec.KubeDNS.ServerIP - NodeLocalDNS.ClusterIP = "__PILLAR__CLUSTER__DNS__" - } + } else if NodeLocalDNS.Enabled && NodeLocalDNS.LocalIP == "" { + NodeLocalDNS.LocalIP = "169.254.20.10" } return nil diff --git a/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template index 4797317d9a11c..e74e7ce084026 100644 --- a/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template +++ b/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template @@ -42,7 +42,7 @@ metadata: addonmanager.kubernetes.io/mode: Reconcile data: Corefile: | - {{ .KubeDNS.NodeLocalDNS.Domain }}:53 { + {{ KubeDNS.Domain }}:53 { errors cache { success 9984 30 @@ -50,20 +50,20 @@ data: } reload loop - bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }} - forward . {{ .KubeDNS.NodeLocalDNS.ClusterIP }} { + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { force_tcp } prometheus :9253 - health {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }}:8080 + health {{ KubeDNS.NodeLocalDNS.LocalIP }}:8080 } in-addr.arpa:53 { errors cache 30 reload loop - bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }} - forward . {{ .KubeDNS.NodeLocalDNS.ClusterIP }} { + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { force_tcp } prometheus :9253 @@ -73,8 +73,8 @@ data: cache 30 reload loop - bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }} - forward . {{ .KubeDNS.NodeLocalDNS.ClusterIP }} { + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { force_tcp } prometheus :9253 @@ -84,7 +84,7 @@ data: cache 30 reload loop - bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }} + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} forward . __PILLAR__UPSTREAM__SERVERS__ { force_tcp } @@ -133,10 +133,10 @@ spec: requests: cpu: 25m memory: 5Mi - {{ if .KubeDNS.NodeLocalDNS.ServerIP }} - args: [ "-localip", "{{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }},{{ .KubeDNS.NodeLocalDNS.ServerIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + {{ if NodeLocalDNSServerIP }} + args: [ "-localip", "{{ .KubeDNS.NodeLocalDNS.LocalIP }},{{ NodeLocalDNSServerIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] {{ else }} - args: [ "-localip", "{{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + args: [ "-localip", "{{ .KubeDNS.NodeLocalDNS.LocalIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] {{ end }} securityContext: privileged: true @@ -152,7 +152,7 @@ spec: protocol: TCP livenessProbe: httpGet: - host: {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} + host: {{ .KubeDNS.NodeLocalDNS.LocalIP }} path: /health port: 8080 initialDelaySeconds: 60 diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 4e4fec025f1c6..060cacbe21f80 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -97,6 +97,21 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS return tf.cluster.Spec.KubeDNS } + dest["NodeLocalDNSClusterIP"] = func() string { + if tf.cluster.Spec.KubeProxy.ProxyMode == "ipvs" { + return tf.cluster.Spec.KubeDNS.ServerIP + } else { + return "__PILLAR__CLUSTER__DNS__" + } + } + dest["NodeLocalDNSServerIP"] = func() string { + if tf.cluster.Spec.KubeProxy.ProxyMode == "ipvs" { + return "" + } else { + return tf.cluster.Spec.KubeDNS.ServerIP + } + } + dest["KopsControllerArgv"] = tf.KopsControllerArgv dest["KopsControllerConfig"] = tf.KopsControllerConfig dest["DnsControllerArgv"] = tf.DnsControllerArgv From f5a5db4ff3057250156485b445e7b30f8534da0a Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Fri, 10 Apr 2020 11:29:49 +0200 Subject: [PATCH 5/5] Verify bindata --- upup/models/bindata.go | 203 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) diff --git a/upup/models/bindata.go b/upup/models/bindata.go index b4d0ccbe0c57c..f837072c6ff84 100644 --- a/upup/models/bindata.go +++ b/upup/models/bindata.go @@ -52,6 +52,7 @@ // upup/models/cloudup/resources/addons/networking.weave/k8s-1.8.yaml.template // upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template // upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.12.yaml.template +// upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template // upup/models/cloudup/resources/addons/openstack.addons.k8s.io/BUILD.bazel // upup/models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.11.yaml.template // upup/models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.13.yaml.template @@ -15371,6 +15372,204 @@ func cloudupResourcesAddonsNodeAuthorizerAddonsK8sIoK8s112YamlTemplate() (*asset return a, nil } +var _cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplate = []byte(`# Vendored from https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-local-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns-upstream + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeDNSUpstream" +spec: + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-local-dns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + Corefile: | + {{ KubeDNS.Domain }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { + force_tcp + } + prometheus :9253 + health {{ KubeDNS.NodeLocalDNS.LocalIP }}:8080 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { + force_tcp + } + prometheus :9253 + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { + force_tcp + } + prometheus :9253 + } + .:53 { + errors + cache 30 + reload + loop + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . __PILLAR__UPSTREAM__SERVERS__ { + force_tcp + } + prometheus :9253 + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-local-dns + namespace: kube-system + labels: + k8s-app: node-local-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + k8s-app: node-local-dns + template: + metadata: + labels: + k8s-app: node-local-dns + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: node-local-dns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + containers: + - name: node-cache + image: k8s.gcr.io/k8s-dns-node-cache:1.15.10 + resources: + requests: + cpu: 25m + memory: 5Mi + {{ if NodeLocalDNSServerIP }} + args: [ "-localip", "{{ .KubeDNS.NodeLocalDNS.LocalIP }},{{ NodeLocalDNSServerIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + {{ else }} + args: [ "-localip", "{{ .KubeDNS.NodeLocalDNS.LocalIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + {{ end }} + securityContext: + privileged: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + host: {{ .KubeDNS.NodeLocalDNS.LocalIP }} + path: /health + port: 8080 + initialDelaySeconds: 60 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - name: config-volume + mountPath: /etc/coredns + - name: kube-dns-config + mountPath: /etc/kube-dns + volumes: + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + - name: config-volume + configMap: + name: node-local-dns + items: + - key: Corefile + path: Corefile.base`) + +func cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplateBytes() ([]byte, error) { + return _cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplate, nil +} + +func cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplate() (*asset, error) { + bytes, err := cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplateBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _cloudupResourcesAddonsOpenstackAddonsK8sIoBuildBazel = []byte(`filegroup( name = "exported_testdata", srcs = glob(["**"]), @@ -17035,6 +17234,7 @@ var _bindata = map[string]func() (*asset, error){ "cloudup/resources/addons/networking.weave/k8s-1.8.yaml.template": cloudupResourcesAddonsNetworkingWeaveK8s18YamlTemplate, "cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template": cloudupResourcesAddonsNodeAuthorizerAddonsK8sIoK8s110YamlTemplate, "cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.12.yaml.template": cloudupResourcesAddonsNodeAuthorizerAddonsK8sIoK8s112YamlTemplate, + "cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template": cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplate, "cloudup/resources/addons/openstack.addons.k8s.io/BUILD.bazel": cloudupResourcesAddonsOpenstackAddonsK8sIoBuildBazel, "cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.11.yaml.template": cloudupResourcesAddonsOpenstackAddonsK8sIoK8s111YamlTemplate, "cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.13.yaml.template": cloudupResourcesAddonsOpenstackAddonsK8sIoK8s113YamlTemplate, @@ -17193,6 +17393,9 @@ var _bintree = &bintree{nil, map[string]*bintree{ "k8s-1.10.yaml.template": {cloudupResourcesAddonsNodeAuthorizerAddonsK8sIoK8s110YamlTemplate, map[string]*bintree{}}, "k8s-1.12.yaml.template": {cloudupResourcesAddonsNodeAuthorizerAddonsK8sIoK8s112YamlTemplate, map[string]*bintree{}}, }}, + "nodelocaldns.addons.k8s.io": {nil, map[string]*bintree{ + "k8s-1.12.yaml.template": {cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplate, map[string]*bintree{}}, + }}, "openstack.addons.k8s.io": {nil, map[string]*bintree{ "BUILD.bazel": {cloudupResourcesAddonsOpenstackAddonsK8sIoBuildBazel, map[string]*bintree{}}, "k8s-1.11.yaml.template": {cloudupResourcesAddonsOpenstackAddonsK8sIoK8s111YamlTemplate, map[string]*bintree{}},