diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index 0ab8981d292ee..eb23177e08633 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -543,6 +543,26 @@ spec: **Note:** If you are upgrading to CoreDNS, kube-dns will be left in place and must be removed manually (you can scale the kube-dns and kube-dns-autoscaler deployments in the `kube-system` namespace to 0 as a starting point). The `kube-dns` Service itself should be left in place, as this retains the ClusterIP and eliminates the possibility of DNS outages in your cluster. If you would like to continue autoscaling, update the `kube-dns-autoscaler` Deployment container command for `--target=Deployment/kube-dns` to be `--target=Deployment/coredns`. +If you are using CoreDNS, you can enable NodeLocal DNSCache. It is used to improve improve the Cluster DNS performance by running a dns caching agent on cluster nodes as a DaemonSet. + +```yaml +spec: + kubeDNS: + provider: CoreDNS + nodeLocalDNS: + enabled: true +``` + +If you are using kube-proxy in ipvs mode or Cilium as CNI, you have to set the nodeLocalDNS as ClusterDNS. + +```yaml +spec: + kubelet: + clusterDNS: 169.254.20.10 + masterKubelet: + clusterDNS: 169.254.20.10 +``` + ### kubeControllerManager This block contains configurations for the `controller-manager`. diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 64c197e1082d9..42a48128fdb13 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -1611,6 +1611,20 @@ spec: dns container in the cluster. Default 70m. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + nodeLocalDNS: + description: NodeLocalDNS specifies the configuration for the + node-local-dns addon + properties: + enabled: + description: Disable indicates we do not wish to run the node-local-dns + addon + type: boolean + localIP: + description: Local listen IP address. It can be any IP in + the 169.254.20.0/16 space or any other IP address that can + be guaranteed to not collide with any existing IP. + type: string + type: object provider: description: Provider indicates whether CoreDNS or kube-dns will be the default service discovery. diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 5896b2c86fb45..ba3c1c407f8ad 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -400,6 +400,16 @@ type KubeDNSConfig struct { CPURequest *resource.Quantity `json:"cpuRequest,omitempty"` // MemoryLimit specifies the memory limit of each dns container in the cluster. Default 170m. MemoryLimit *resource.Quantity `json:"memoryLimit,omitempty"` + // NodeLocalDNS specifies the configuration for the node-local-dns addon + NodeLocalDNS *NodeLocalDNSConfig `json:"nodeLocalDNS,omitempty"` +} + +// NodeLocalDNSConfig are options of the node-local-dns +type NodeLocalDNSConfig struct { + // Disable indicates we do not wish to run the node-local-dns addon + Enabled bool `json:"enabled,omitempty"` + // Local listen IP address. It can be any IP in the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP. + LocalIP string `json:"localIP,omitempty"` } // ExternalDNSConfig are options of the dns-controller diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index af05ba7d7178a..53ec1a1069a02 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -398,6 +398,16 @@ type KubeDNSConfig struct { CPURequest *resource.Quantity `json:"cpuRequest,omitempty"` // MemoryLimit specifies the memory limit of each dns container in the cluster. Default 170m. MemoryLimit *resource.Quantity `json:"memoryLimit,omitempty"` + // NodeLocalDNS specifies the configuration for the node-local-dns addon + NodeLocalDNS *NodeLocalDNSConfig `json:"nodeLocalDNS,omitempty"` +} + +// NodeLocalDNSConfig are options of the node-local-dns +type NodeLocalDNSConfig struct { + // Disable indicates we do not wish to run the node-local-dns addon + Enabled bool `json:"enabled,omitempty"` + // Local listen IP address. It can be any IP in the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP. + LocalIP string `json:"localIP,omitempty"` } // ExternalDNSConfig are options of the dns-controller diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index f55708fc474b2..d6142016455a9 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -703,6 +703,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*NodeLocalDNSConfig)(nil), (*kops.NodeLocalDNSConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(a.(*NodeLocalDNSConfig), b.(*kops.NodeLocalDNSConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.NodeLocalDNSConfig)(nil), (*NodeLocalDNSConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(a.(*kops.NodeLocalDNSConfig), b.(*NodeLocalDNSConfig), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*OpenstackBlockStorageConfig)(nil), (*kops.OpenstackBlockStorageConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(a.(*OpenstackBlockStorageConfig), b.(*kops.OpenstackBlockStorageConfig), scope) }); err != nil { @@ -3965,6 +3975,15 @@ func autoConvert_v1alpha2_KubeDNSConfig_To_kops_KubeDNSConfig(in *KubeDNSConfig, out.MemoryRequest = in.MemoryRequest out.CPURequest = in.CPURequest out.MemoryLimit = in.MemoryLimit + if in.NodeLocalDNS != nil { + in, out := &in.NodeLocalDNS, &out.NodeLocalDNS + *out = new(kops.NodeLocalDNSConfig) + if err := Convert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(*in, *out, s); err != nil { + return err + } + } else { + out.NodeLocalDNS = nil + } return nil } @@ -3988,6 +4007,15 @@ func autoConvert_kops_KubeDNSConfig_To_v1alpha2_KubeDNSConfig(in *kops.KubeDNSCo out.MemoryRequest = in.MemoryRequest out.CPURequest = in.CPURequest out.MemoryLimit = in.MemoryLimit + if in.NodeLocalDNS != nil { + in, out := &in.NodeLocalDNS, &out.NodeLocalDNS + *out = new(NodeLocalDNSConfig) + if err := Convert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(*in, *out, s); err != nil { + return err + } + } else { + out.NodeLocalDNS = nil + } return nil } @@ -4808,6 +4836,28 @@ func Convert_kops_NodeAuthorizerSpec_To_v1alpha2_NodeAuthorizerSpec(in *kops.Nod return autoConvert_kops_NodeAuthorizerSpec_To_v1alpha2_NodeAuthorizerSpec(in, out, s) } +func autoConvert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLocalDNSConfig, out *kops.NodeLocalDNSConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + out.LocalIP = in.LocalIP + return nil +} + +// Convert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig is an autogenerated conversion function. +func Convert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLocalDNSConfig, out *kops.NodeLocalDNSConfig, s conversion.Scope) error { + return autoConvert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in, out, s) +} + +func autoConvert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(in *kops.NodeLocalDNSConfig, out *NodeLocalDNSConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + out.LocalIP = in.LocalIP + return nil +} + +// Convert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig is an autogenerated conversion function. +func Convert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(in *kops.NodeLocalDNSConfig, out *NodeLocalDNSConfig, s conversion.Scope) error { + return autoConvert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(in, out, s) +} + func autoConvert_v1alpha2_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(in *OpenstackBlockStorageConfig, out *kops.OpenstackBlockStorageConfig, s conversion.Scope) error { out.Version = in.Version out.IgnoreAZ = in.IgnoreAZ diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index e58ab0775a55b..d893ebf068dd2 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -2451,6 +2451,11 @@ func (in *KubeDNSConfig) DeepCopyInto(out *KubeDNSConfig) { x := (*in).DeepCopy() *out = &x } + if in.NodeLocalDNS != nil { + in, out := &in.NodeLocalDNS, &out.NodeLocalDNS + *out = new(NodeLocalDNSConfig) + **out = **in + } return } @@ -3175,6 +3180,22 @@ func (in *NodeAuthorizerSpec) DeepCopy() *NodeAuthorizerSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeLocalDNSConfig) DeepCopyInto(out *NodeLocalDNSConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeLocalDNSConfig. +func (in *NodeLocalDNSConfig) DeepCopy() *NodeLocalDNSConfig { + if in == nil { + return nil + } + out := new(NodeLocalDNSConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenstackBlockStorageConfig) DeepCopyInto(out *OpenstackBlockStorageConfig) { *out = *in diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go index 69fc536629b54..22c68fbbcff8d 100644 --- a/pkg/apis/kops/validation/legacy.go +++ b/pkg/apis/kops/validation/legacy.go @@ -277,14 +277,23 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList { allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "serverIP"), fmt.Sprintf("ServiceClusterIPRange %q must contain the DNS Server IP %q", c.Spec.ServiceClusterIPRange, address))) } if !featureflag.ExperimentalClusterDNS.Enabled() { - if c.Spec.Kubelet != nil && c.Spec.Kubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP { - allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "serverIP"), "Kubelet ClusterDNS did not match cluster kubeDNS.serverIP")) + if isExperimentalClusterDNS(c.Spec.Kubelet, c.Spec.KubeDNS) { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubelet", "clusterDNS"), "Kubelet ClusterDNS did not match cluster kubeDNS.serverIP or nodeLocalDNS.localIP")) } - if c.Spec.MasterKubelet != nil && c.Spec.MasterKubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP { - allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "serverIP"), "MasterKubelet ClusterDNS did not match cluster kubeDNS.serverIP")) + if isExperimentalClusterDNS(c.Spec.MasterKubelet, c.Spec.KubeDNS) { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("masterKubelet", "clusterDNS"), "MasterKubelet ClusterDNS did not match cluster kubeDNS.serverIP or nodeLocalDNS.localIP")) } } } + + // @ check that NodeLocalDNS addon is configured correctly + if c.Spec.KubeDNS.NodeLocalDNS != nil && c.Spec.KubeDNS.NodeLocalDNS.Enabled { + if c.Spec.KubeDNS.Provider != "CoreDNS" { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "provider"), "KubeDNS provider must be set to CoreDNS if NodeLocalDNS addon is enabled")) + } + + allErrs = append(allErrs, validateNodeLocalDNS(&c.Spec, fieldSpec.Child("spec"))...) + } } // @check the nameservers are valid @@ -705,3 +714,9 @@ func validateKubelet(k *kops.KubeletConfigSpec, c *kops.Cluster, kubeletPath *fi } return allErrs } + +func isExperimentalClusterDNS(k *kops.KubeletConfigSpec, dns *kops.KubeDNSConfig) bool { + + return k != nil && k.ClusterDNS != dns.ServerIP && dns.NodeLocalDNS != nil && k.ClusterDNS != dns.NodeLocalDNS.LocalIP + +} diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index d30e5d72d108a..77ad44b9b2894 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -646,3 +646,27 @@ func validateRollingUpdate(rollingUpdate *kops.RollingUpdate, fldpath *field.Pat return allErrs } + +func validateNodeLocalDNS(spec *kops.ClusterSpec, fldpath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if spec.KubeDNS.NodeLocalDNS.LocalIP != "" { + address := spec.KubeDNS.NodeLocalDNS.LocalIP + ip := net.ParseIP(address) + if ip == nil { + allErrs = append(allErrs, field.Invalid(fldpath.Child("kubeDNS", "nodeLocalDNS", "localIP"), address, "Cluster had an invalid kubeDNS.nodeLocalDNS.localIP")) + } + } + + if (spec.KubeProxy != nil && spec.KubeProxy.ProxyMode == "ipvs") || (spec.Networking != nil && spec.Networking.Cilium != nil) { + if spec.Kubelet != nil && spec.Kubelet.ClusterDNS != spec.KubeDNS.NodeLocalDNS.LocalIP { + allErrs = append(allErrs, field.Forbidden(fldpath.Child("kubelet", "clusterDNS"), "Kubelet ClusterDNS must be set to the default IP address for LocalIP")) + } + + if spec.MasterKubelet != nil && spec.MasterKubelet.ClusterDNS != spec.KubeDNS.NodeLocalDNS.LocalIP { + allErrs = append(allErrs, field.Forbidden(fldpath.Child("kubelet", "clusterDNS"), "MasterKubelet ClusterDNS must be set to the default IP address for LocalIP")) + } + } + + return allErrs +} diff --git a/pkg/apis/kops/validation/validation_test.go b/pkg/apis/kops/validation/validation_test.go index 037b6bce9ff1d..52ca54d6e2214 100644 --- a/pkg/apis/kops/validation/validation_test.go +++ b/pkg/apis/kops/validation/validation_test.go @@ -534,3 +534,88 @@ func Test_Validate_RollingUpdate(t *testing.T) { func intStr(i intstr.IntOrString) *intstr.IntOrString { return &i } + +func Test_Validate_NodeLocalDNS(t *testing.T) { + grid := []struct { + Input kops.ClusterSpec + ExpectedErrors []string + }{ + { + Input: kops.ClusterSpec{ + KubeProxy: &kops.KubeProxyConfig{ + ProxyMode: "iptables", + }, + KubeDNS: &kops.KubeDNSConfig{ + Provider: "CoreDNS", + NodeLocalDNS: &kops.NodeLocalDNSConfig{ + Enabled: true, + }, + }, + }, + ExpectedErrors: []string{}, + }, + { + Input: kops.ClusterSpec{ + Kubelet: &kops.KubeletConfigSpec{ + ClusterDNS: "100.64.0.10", + }, + KubeProxy: &kops.KubeProxyConfig{ + ProxyMode: "ipvs", + }, + KubeDNS: &kops.KubeDNSConfig{ + Provider: "CoreDNS", + NodeLocalDNS: &kops.NodeLocalDNSConfig{ + Enabled: true, + }, + }, + }, + ExpectedErrors: []string{"Forbidden::spec.kubelet.clusterDNS"}, + }, + { + Input: kops.ClusterSpec{ + Kubelet: &kops.KubeletConfigSpec{ + ClusterDNS: "100.64.0.10", + }, + KubeProxy: &kops.KubeProxyConfig{ + ProxyMode: "ipvs", + }, + KubeDNS: &kops.KubeDNSConfig{ + Provider: "CoreDNS", + NodeLocalDNS: &kops.NodeLocalDNSConfig{ + Enabled: true, + }, + }, + Networking: &kops.NetworkingSpec{ + Cilium: &kops.CiliumNetworkingSpec{}, + }, + }, + ExpectedErrors: []string{"Forbidden::spec.kubelet.clusterDNS"}, + }, + { + Input: kops.ClusterSpec{ + Kubelet: &kops.KubeletConfigSpec{ + ClusterDNS: "169.254.20.10", + }, + KubeProxy: &kops.KubeProxyConfig{ + ProxyMode: "iptables", + }, + KubeDNS: &kops.KubeDNSConfig{ + Provider: "CoreDNS", + NodeLocalDNS: &kops.NodeLocalDNSConfig{ + Enabled: true, + LocalIP: "169.254.20.10", + }, + }, + Networking: &kops.NetworkingSpec{ + Cilium: &kops.CiliumNetworkingSpec{}, + }, + }, + ExpectedErrors: []string{}, + }, + } + + for _, g := range grid { + errs := validateNodeLocalDNS(&g.Input, field.NewPath("spec")) + testErrors(t, g.Input, errs, g.ExpectedErrors) + } +} diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 5f00976e9e65b..e169e05c1502b 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -2633,6 +2633,11 @@ func (in *KubeDNSConfig) DeepCopyInto(out *KubeDNSConfig) { x := (*in).DeepCopy() *out = &x } + if in.NodeLocalDNS != nil { + in, out := &in.NodeLocalDNS, &out.NodeLocalDNS + *out = new(NodeLocalDNSConfig) + **out = **in + } return } @@ -3373,6 +3378,22 @@ func (in *NodeAuthorizerSpec) DeepCopy() *NodeAuthorizerSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeLocalDNSConfig) DeepCopyInto(out *NodeLocalDNSConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeLocalDNSConfig. +func (in *NodeLocalDNSConfig) DeepCopy() *NodeLocalDNSConfig { + if in == nil { + return nil + } + out := new(NodeLocalDNSConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NoopStatusStore) DeepCopyInto(out *NoopStatusStore) { *out = *in diff --git a/pkg/model/components/kubedns.go b/pkg/model/components/kubedns.go index 3c8ba3ce343b1..3e56fdad8ac96 100644 --- a/pkg/model/components/kubedns.go +++ b/pkg/model/components/kubedns.go @@ -74,5 +74,13 @@ func (b *KubeDnsOptionsBuilder) BuildOptions(o interface{}) error { clusterSpec.KubeDNS.MemoryLimit = &defaultMemoryLimit } + NodeLocalDNS := clusterSpec.KubeDNS.NodeLocalDNS + if NodeLocalDNS == nil { + NodeLocalDNS = &kops.NodeLocalDNSConfig{} + NodeLocalDNS.Enabled = false + } else if NodeLocalDNS.Enabled && NodeLocalDNS.LocalIP == "" { + NodeLocalDNS.LocalIP = "169.254.20.10" + } + return nil } diff --git a/upup/models/bindata.go b/upup/models/bindata.go index b4d0ccbe0c57c..f837072c6ff84 100644 --- a/upup/models/bindata.go +++ b/upup/models/bindata.go @@ -52,6 +52,7 @@ // upup/models/cloudup/resources/addons/networking.weave/k8s-1.8.yaml.template // upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template // upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.12.yaml.template +// upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template // upup/models/cloudup/resources/addons/openstack.addons.k8s.io/BUILD.bazel // upup/models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.11.yaml.template // upup/models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.13.yaml.template @@ -15371,6 +15372,204 @@ func cloudupResourcesAddonsNodeAuthorizerAddonsK8sIoK8s112YamlTemplate() (*asset return a, nil } +var _cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplate = []byte(`# Vendored from https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-local-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns-upstream + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeDNSUpstream" +spec: + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-local-dns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + Corefile: | + {{ KubeDNS.Domain }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { + force_tcp + } + prometheus :9253 + health {{ KubeDNS.NodeLocalDNS.LocalIP }}:8080 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { + force_tcp + } + prometheus :9253 + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { + force_tcp + } + prometheus :9253 + } + .:53 { + errors + cache 30 + reload + loop + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . __PILLAR__UPSTREAM__SERVERS__ { + force_tcp + } + prometheus :9253 + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-local-dns + namespace: kube-system + labels: + k8s-app: node-local-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + k8s-app: node-local-dns + template: + metadata: + labels: + k8s-app: node-local-dns + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: node-local-dns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + containers: + - name: node-cache + image: k8s.gcr.io/k8s-dns-node-cache:1.15.10 + resources: + requests: + cpu: 25m + memory: 5Mi + {{ if NodeLocalDNSServerIP }} + args: [ "-localip", "{{ .KubeDNS.NodeLocalDNS.LocalIP }},{{ NodeLocalDNSServerIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + {{ else }} + args: [ "-localip", "{{ .KubeDNS.NodeLocalDNS.LocalIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + {{ end }} + securityContext: + privileged: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + host: {{ .KubeDNS.NodeLocalDNS.LocalIP }} + path: /health + port: 8080 + initialDelaySeconds: 60 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - name: config-volume + mountPath: /etc/coredns + - name: kube-dns-config + mountPath: /etc/kube-dns + volumes: + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + - name: config-volume + configMap: + name: node-local-dns + items: + - key: Corefile + path: Corefile.base`) + +func cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplateBytes() ([]byte, error) { + return _cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplate, nil +} + +func cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplate() (*asset, error) { + bytes, err := cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplateBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _cloudupResourcesAddonsOpenstackAddonsK8sIoBuildBazel = []byte(`filegroup( name = "exported_testdata", srcs = glob(["**"]), @@ -17035,6 +17234,7 @@ var _bindata = map[string]func() (*asset, error){ "cloudup/resources/addons/networking.weave/k8s-1.8.yaml.template": cloudupResourcesAddonsNetworkingWeaveK8s18YamlTemplate, "cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template": cloudupResourcesAddonsNodeAuthorizerAddonsK8sIoK8s110YamlTemplate, "cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.12.yaml.template": cloudupResourcesAddonsNodeAuthorizerAddonsK8sIoK8s112YamlTemplate, + "cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template": cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplate, "cloudup/resources/addons/openstack.addons.k8s.io/BUILD.bazel": cloudupResourcesAddonsOpenstackAddonsK8sIoBuildBazel, "cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.11.yaml.template": cloudupResourcesAddonsOpenstackAddonsK8sIoK8s111YamlTemplate, "cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.13.yaml.template": cloudupResourcesAddonsOpenstackAddonsK8sIoK8s113YamlTemplate, @@ -17193,6 +17393,9 @@ var _bintree = &bintree{nil, map[string]*bintree{ "k8s-1.10.yaml.template": {cloudupResourcesAddonsNodeAuthorizerAddonsK8sIoK8s110YamlTemplate, map[string]*bintree{}}, "k8s-1.12.yaml.template": {cloudupResourcesAddonsNodeAuthorizerAddonsK8sIoK8s112YamlTemplate, map[string]*bintree{}}, }}, + "nodelocaldns.addons.k8s.io": {nil, map[string]*bintree{ + "k8s-1.12.yaml.template": {cloudupResourcesAddonsNodelocaldnsAddonsK8sIoK8s112YamlTemplate, map[string]*bintree{}}, + }}, "openstack.addons.k8s.io": {nil, map[string]*bintree{ "BUILD.bazel": {cloudupResourcesAddonsOpenstackAddonsK8sIoBuildBazel, map[string]*bintree{}}, "k8s-1.11.yaml.template": {cloudupResourcesAddonsOpenstackAddonsK8sIoK8s111YamlTemplate, map[string]*bintree{}}, diff --git a/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template new file mode 100644 index 0000000000000..e74e7ce084026 --- /dev/null +++ b/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template @@ -0,0 +1,182 @@ +# Vendored from https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-local-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns-upstream + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeDNSUpstream" +spec: + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-local-dns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + Corefile: | + {{ KubeDNS.Domain }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { + force_tcp + } + prometheus :9253 + health {{ KubeDNS.NodeLocalDNS.LocalIP }}:8080 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { + force_tcp + } + prometheus :9253 + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { + force_tcp + } + prometheus :9253 + } + .:53 { + errors + cache 30 + reload + loop + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . __PILLAR__UPSTREAM__SERVERS__ { + force_tcp + } + prometheus :9253 + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-local-dns + namespace: kube-system + labels: + k8s-app: node-local-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + k8s-app: node-local-dns + template: + metadata: + labels: + k8s-app: node-local-dns + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: node-local-dns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + containers: + - name: node-cache + image: k8s.gcr.io/k8s-dns-node-cache:1.15.10 + resources: + requests: + cpu: 25m + memory: 5Mi + {{ if NodeLocalDNSServerIP }} + args: [ "-localip", "{{ .KubeDNS.NodeLocalDNS.LocalIP }},{{ NodeLocalDNSServerIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + {{ else }} + args: [ "-localip", "{{ .KubeDNS.NodeLocalDNS.LocalIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + {{ end }} + securityContext: + privileged: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + host: {{ .KubeDNS.NodeLocalDNS.LocalIP }} + path: /health + port: 8080 + initialDelaySeconds: 60 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - name: config-volume + mountPath: /etc/coredns + - name: kube-dns-config + mountPath: /etc/kube-dns + volumes: + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + - name: config-volume + configMap: + name: node-local-dns + items: + - key: Corefile + path: Corefile.base \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index e47f6ca1541b3..df2a40b9e44b6 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -451,6 +451,29 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { } } + // @check the node-local-dns has not been disabled + NodeLocalDNS := b.cluster.Spec.KubeDNS.NodeLocalDNS + if kubeDNS.Provider == "CoreDNS" && NodeLocalDNS != nil && NodeLocalDNS.Enabled { + { + key := "nodelocaldns.addons.k8s.io" + version := "1.18.0" + + { + location := key + "/k8s-1.12.yaml" + id := "k8s-1.12" + + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(version), + Selector: map[string]string{"k8s-addon": key}, + Manifest: fi.String(location), + KubernetesVersion: ">=1.12.0", + Id: id, + }) + } + } + } + if kops.CloudProviderID(b.cluster.Spec.CloudProvider) == kops.CloudProviderAWS { key := "storage-aws.addons.k8s.io" version := "1.15.0" diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 4e4fec025f1c6..060cacbe21f80 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -97,6 +97,21 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS return tf.cluster.Spec.KubeDNS } + dest["NodeLocalDNSClusterIP"] = func() string { + if tf.cluster.Spec.KubeProxy.ProxyMode == "ipvs" { + return tf.cluster.Spec.KubeDNS.ServerIP + } else { + return "__PILLAR__CLUSTER__DNS__" + } + } + dest["NodeLocalDNSServerIP"] = func() string { + if tf.cluster.Spec.KubeProxy.ProxyMode == "ipvs" { + return "" + } else { + return tf.cluster.Spec.KubeDNS.ServerIP + } + } + dest["KopsControllerArgv"] = tf.KopsControllerArgv dest["KopsControllerConfig"] = tf.KopsControllerConfig dest["DnsControllerArgv"] = tf.DnsControllerArgv