From 0a662aeea50a5a5f561a06a31f4106445bc19f07 Mon Sep 17 00:00:00 2001 From: ONE7live Date: Thu, 11 Jul 2024 13:40:57 +0800 Subject: [PATCH] feat: add pluggable component installation method Signed-off-by: ONE7live --- charts/plugins/node-local-dns/.helmignore | 23 ++ charts/plugins/node-local-dns/Chart.yaml | 22 ++ charts/plugins/node-local-dns/README.md | 33 +++ .../node-local-dns/templates/_helpers.tpl | 7 + .../node-local-dns/templates/configmap.yaml | 56 +++++ .../node-local-dns/templates/daemonset.yaml | 82 +++++++ .../node-local-dns/templates/service.yaml | 44 ++++ .../templates/serviceaccount.yaml | 9 + charts/plugins/node-local-dns/values.yaml | 27 +++ cmd/kubenest/operator/app/operator.go | 11 +- .../v1alpha1/virtualclusterplugin_types.go | 18 ++ pkg/kubenest/constants/constant.go | 1 + .../vc_plugin_controller.go | 203 ++++++++++++++++ .../plugins/manifest_node-local-dns.go | 227 ++++++++++++++++++ 14 files changed, 762 insertions(+), 1 deletion(-) create mode 100644 charts/plugins/node-local-dns/.helmignore create mode 100644 charts/plugins/node-local-dns/Chart.yaml create mode 100644 charts/plugins/node-local-dns/README.md create mode 100644 charts/plugins/node-local-dns/templates/_helpers.tpl create mode 100644 charts/plugins/node-local-dns/templates/configmap.yaml create mode 100644 charts/plugins/node-local-dns/templates/daemonset.yaml create mode 100644 charts/plugins/node-local-dns/templates/service.yaml create mode 100644 charts/plugins/node-local-dns/templates/serviceaccount.yaml create mode 100644 charts/plugins/node-local-dns/values.yaml create mode 100644 pkg/kubenest/controller/virtualcluster.plugin.controller/vc_plugin_controller.go create mode 100644 pkg/kubenest/manifest/plugins/manifest_node-local-dns.go diff --git a/charts/plugins/node-local-dns/.helmignore b/charts/plugins/node-local-dns/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/plugins/node-local-dns/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/plugins/node-local-dns/Chart.yaml b/charts/plugins/node-local-dns/Chart.yaml new file mode 100644 index 000000000..bfc2af184 --- /dev/null +++ b/charts/plugins/node-local-dns/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v1alpha1 +name: node-local-dns +namespace: kube-system +description: A Helm chart for Kubenest plugin Node-local-dns + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 0.0.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. +appVersion: latest diff --git a/charts/plugins/node-local-dns/README.md b/charts/plugins/node-local-dns/README.md new file mode 100644 index 000000000..548f45251 --- /dev/null +++ b/charts/plugins/node-local-dns/README.md @@ -0,0 +1,33 @@ +# Node-Local-DNS + +Kosmos-kubenest plugin NodeLocalDNS helm chart + +## Summary + +The chart install NodeLocalDNS set according to . + +It is designed to work both with Iptables and IPVS setup. + +Latest available `node-local-dns` image can be found at [node-local-dns google container repository](https://console.cloud.google.com/gcr/images/google-containers/GLOBAL/k8s-dns-node-cache) + +## Values + +| Key | Type | Default | Description | +|---------------------------|------|--------------------------------------------|-------------| +| image.repository | string | `"registry.k8s.io/dns/k8s-dns-node-cache"` | | +| image.version | string | `"1.23.1"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| config.domain | string | `"cluster.local"` | | +| config.kubeDNS | string | `"xxx.xxx.xxx.xxx"` | | +| config.localDNS | string | `"xxx.xxx.xxx.xxx"` | | +| config.clusterDNS | string | `"xxx.xxx.xxx.xxx"` | | +| resources.requests.cpu | string | `"25m"` | | +| resources.requests.memory | string | `"5Mi"` | | +| tolerations[0].key | string | `"CriticalAddonsOnly"` | | +| tolerations[0].operator | string | `"Exists"` | | +| tolerations[1].effect | string | `"NoExecute"` | | +| tolerations[1].operator | string | `"Exists"` | | +| tolerations[2].effect | string | `"NoSchedule"` | | +| tolerations[2].operator | string | `"Exists"` | | +| nodeSelector | object | `{}` | | +| affinity | object | `{}` | | diff --git a/charts/plugins/node-local-dns/templates/_helpers.tpl b/charts/plugins/node-local-dns/templates/_helpers.tpl new file mode 100644 index 000000000..7966482ef --- /dev/null +++ b/charts/plugins/node-local-dns/templates/_helpers.tpl @@ -0,0 +1,7 @@ +{{- define "node-local-dns.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "node-local-dns.namespace" -}} +{{- default .Chart.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" }} +{{- end }} diff --git a/charts/plugins/node-local-dns/templates/configmap.yaml b/charts/plugins/node-local-dns/templates/configmap.yaml new file mode 100644 index 000000000..e44df3bfd --- /dev/null +++ b/charts/plugins/node-local-dns/templates/configmap.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "node-local-dns.name" . }} + namespace: {{ include "node-local-dns.namespace" . }} + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + Corefile: | + {{ .Values.configMap.domain }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ .Values.configMap.localDNS }} {{ .Values.configMap.kubeDNS }} + forward . {{ .Values.configMap.clusterDNS }} { + force_tcp + } + prometheus :9253 + health {{ .Values.configMap.localDNS }}:8080 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ .Values.configMap.localDNS }} {{ .Values.configMap.kubeDNS }} + forward . {{ .Values.configMap.clusterDNS }} { + force_tcp + } + prometheus :9253 + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ .Values.configMap.localDNS }} {{ .Values.configMap.kubeDNS }} + forward . {{ .Values.configMap.clusterDNS }} { + force_tcp + } + prometheus :9253 + } + .:53 { + errors + cache 30 + reload + loop + bind {{ .Values.configMap.localDNS }} {{ .Values.configMap.kubeDNS }} + forward . __PILLAR__UPSTREAM__SERVERS__ + prometheus :9253 + } \ No newline at end of file diff --git a/charts/plugins/node-local-dns/templates/daemonset.yaml b/charts/plugins/node-local-dns/templates/daemonset.yaml new file mode 100644 index 000000000..40c5ee91a --- /dev/null +++ b/charts/plugins/node-local-dns/templates/daemonset.yaml @@ -0,0 +1,82 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "node-local-dns.name" . }} + namespace: {{ include "node-local-dns.namespace" . }} + labels: + k8s-app: {{ include "node-local-dns.name" . }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + k8s-app: {{ include "node-local-dns.name" . }} + template: + metadata: + labels: + k8s-app: {{ include "node-local-dns.name" . }} + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: {{ include "node-local-dns.name" . }} + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + {{- toYaml .Values.tolerations | nindent 8 }} + containers: + - name: node-cache + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.version }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + args: [ "-localip", "{{ .Values.configMap.localDNS }},{{ .Values.configMap.kubeDNS }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + securityContext: + capabilities: + add: + - NET_ADMIN + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + host: {{ .Values.configMap.localDNS }} + path: /health + port: 8080 + initialDelaySeconds: 60 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - name: config-volume + mountPath: /etc/coredns + - name: kube-dns-config + mountPath: /etc/kube-dns + volumes: + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + - name: config-volume + configMap: + name: {{ include "node-local-dns.name" . }} + items: + - key: Corefile + path: Corefile.base \ No newline at end of file diff --git a/charts/plugins/node-local-dns/templates/service.yaml b/charts/plugins/node-local-dns/templates/service.yaml new file mode 100644 index 000000000..a5eb83f21 --- /dev/null +++ b/charts/plugins/node-local-dns/templates/service.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns-upstream + namespace: {{ include "node-local-dns.namespace" . }} + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeDNSUpstream" +spec: + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns +--- +# A headless service is a service with a service IP but instead of load-balancing it will return the IPs of our associated Pods. +# We use this to expose metrics to Prometheus. +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + labels: + k8s-app: {{ include "node-local-dns.name" . }} + name: {{ include "node-local-dns.name" . }} + namespace: {{ include "node-local-dns.namespace" . }} +spec: + clusterIP: None + ports: + - name: metrics + port: 9253 + targetPort: 9253 + selector: + k8s-app: {{ include "node-local-dns.name" . }} diff --git a/charts/plugins/node-local-dns/templates/serviceaccount.yaml b/charts/plugins/node-local-dns/templates/serviceaccount.yaml new file mode 100644 index 000000000..465264ab4 --- /dev/null +++ b/charts/plugins/node-local-dns/templates/serviceaccount.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "node-local-dns.name" . }} + namespace: {{ include "node-local-dns.namespace" . }} + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile diff --git a/charts/plugins/node-local-dns/values.yaml b/charts/plugins/node-local-dns/values.yaml new file mode 100644 index 000000000..7233b4ebc --- /dev/null +++ b/charts/plugins/node-local-dns/values.yaml @@ -0,0 +1,27 @@ +--- +image: + repository: registry.k8s.io/dns/k8s-dns-node-cache + version: 1.23.1 + pullPolicy: IfNotPresent +--- +config: + domain: xxx.xxx + kubeDNS: xxx.xxx.xxx + localDNS: xxx.xxx.xxx + clusterDNS: xxx.xxx.xxx +--- +tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists +--- +resources: + requests: + cpu: 25m + memory: 5Mi +--- +nodeSelector: {} +affinity: {} diff --git a/cmd/kubenest/operator/app/operator.go b/cmd/kubenest/operator/app/operator.go index 00b0e3b40..477868726 100644 --- a/cmd/kubenest/operator/app/operator.go +++ b/cmd/kubenest/operator/app/operator.go @@ -260,7 +260,6 @@ func run(ctx context.Context, config *config.Config) error { KosmosClient: kosmosClient, EventRecorder: mgr.GetEventRecorderFor(constants.GlobalNodeControllerName), } - if err = GlobalNodeController.SetupWithManager(mgr); err != nil { return fmt.Errorf("error starting %s: %v", constants.GlobalNodeControllerName, err) } @@ -294,6 +293,16 @@ func run(ctx context.Context, config *config.Config) error { } } + //VirtualClusterPluginController := vcpc.VirtualClusterPluginController{ + // Client: mgr.GetClient(), + // RootClientSet: hostKubeClient, + // KosmosClient: kosmosClient, + // EventRecorder: mgr.GetEventRecorderFor(constants.PluginControllerName), + //} + //if err = VirtualClusterPluginController.SetupWithManager(mgr); err != nil { + // return fmt.Errorf("error starting %s: %v", constants.PluginControllerName, err) + //} + if err := mgr.Start(ctx); err != nil { return fmt.Errorf("failed to start controller manager: %v", err) } diff --git a/pkg/apis/kosmos/v1alpha1/virtualclusterplugin_types.go b/pkg/apis/kosmos/v1alpha1/virtualclusterplugin_types.go index ad8b26bde..9bfc0643c 100644 --- a/pkg/apis/kosmos/v1alpha1/virtualclusterplugin_types.go +++ b/pkg/apis/kosmos/v1alpha1/virtualclusterplugin_types.go @@ -51,6 +51,24 @@ type Chart struct { type Yaml struct { // +required Path Storage `json:"path"` + + // +optional + Domain string `json:"domain"` + + // +optional + KubeDNS string `json:"kubeDNS"` + + // +optional + LocalDNS string `json:"localDNS"` + + // +optional + ClusterDNS string `json:"clusterDNS"` + + // +optional + ImageRepository string `json:"imageRepository"` + + // +optional + Version string `json:"version"` } type Storage struct { diff --git a/pkg/kubenest/constants/constant.go b/pkg/kubenest/constants/constant.go index 5a1182115..0ab9dd984 100644 --- a/pkg/kubenest/constants/constant.go +++ b/pkg/kubenest/constants/constant.go @@ -9,6 +9,7 @@ import ( const ( InitControllerName = "virtual-cluster-init-controller" NodeControllerName = "virtual-cluster-node-controller" + PluginControllerName = "virtual-cluster-plugin-controller" GlobalNodeControllerName = "global-node-controller" KosmosJoinControllerName = "kosmos-join-controller" KosmosNs = "kosmos-system" diff --git a/pkg/kubenest/controller/virtualcluster.plugin.controller/vc_plugin_controller.go b/pkg/kubenest/controller/virtualcluster.plugin.controller/vc_plugin_controller.go new file mode 100644 index 000000000..a6d7e63e2 --- /dev/null +++ b/pkg/kubenest/controller/virtualcluster.plugin.controller/vc_plugin_controller.go @@ -0,0 +1,203 @@ +package virtualcluster_plugin_controller + +import ( + "context" + "fmt" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" + "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" + "github.com/kosmos.io/kosmos/pkg/kubenest/constants" + "github.com/kosmos.io/kosmos/pkg/kubenest/manifest/plugins" + "github.com/kosmos.io/kosmos/pkg/kubenest/util" + "github.com/kosmos.io/kosmos/pkg/utils" +) + +type VirtualClusterPluginController struct { + client.Client + RootClientSet kubernetes.Interface + KosmosClient versioned.Interface + EventRecorder record.EventRecorder +} + +func (vcpc *VirtualClusterPluginController) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + klog.V(4).Infof("============ virtual-cluster-plugin-controller start to reconcile %s ============", request.NamespacedName) + defer klog.V(4).Infof("============ virtual-cluster-plugin-controller finish to reconcile %s ============", request.NamespacedName) + + var vc v1alpha1.VirtualCluster + if err := vcpc.Get(ctx, request.NamespacedName, &vc); err != nil { + klog.V(4).Infof("virtual-cluster-plugin-controller: can not found %s", request.NamespacedName) + return reconcile.Result{}, nil + } + + if vc.Status.Phase != v1alpha1.AllNodeReady { + // wait virtual cluster all node ready + return reconcile.Result{RequeueAfter: utils.DefaultRequeueTime}, nil + } + + for _, plugin := range vc.Spec.PluginSet.Enabled { + vcp, err := vcpc.KosmosClient.KosmosV1alpha1().VirtualClusterPlugins(vc.Namespace).Get(ctx, plugin.Name, metav1.GetOptions{}) + if err != nil { + klog.Errorf("virtual-cluster-plugin-controller: unable to find plugin %s, skip install", plugin.Name) + continue + } + + if vcp.Spec.PluginSources.Yaml != (v1alpha1.Yaml{}) { + err = vcpc.createPluginExecutorJobByYaml(ctx, vcp) + if err != nil { + klog.Errorf("virtual-cluster-plugin-controller: unable to create plugin executor job, skip install %s", plugin.Name) + continue + } else { + klog.Infof("virtual-cluster-plugin-controller: plugin %s executor job create success", plugin.Name) + } + } else { + err = vcpc.createPluginExecutorJobByHelm(ctx, vcp) + if err != nil { + klog.Errorf("virtual-cluster-plugin-controller: unable to create plugin executor job, skip install %s", plugin.Name) + continue + } else { + klog.Infof("virtual-cluster-plugin-controller: plugin %s executor job create success", plugin.Name) + } + } + } + + return reconcile.Result{}, nil +} + +// Function to create a plugin executor job +func (vcpc *VirtualClusterPluginController) createPluginExecutorJobByYaml(ctx context.Context, vcp *v1alpha1.VirtualClusterPlugin) error { + var pluginYamlContent string + switch vcp.Name { + case "node-local-dns": + nodeLocalDNS, err := util.ParseTemplate(plugins.NodeLocalDNSPlugin, struct { + Domain, KubeDNS, LocalDNS, ClusterDNS string + ImageRepository, Version string + }{ + Domain: vcp.Spec.PluginSources.Yaml.Domain, + KubeDNS: vcp.Spec.PluginSources.Yaml.KubeDNS, + LocalDNS: vcp.Spec.PluginSources.Yaml.LocalDNS, + ClusterDNS: vcp.Spec.PluginSources.Yaml.ClusterDNS, + ImageRepository: vcp.Spec.PluginSources.Yaml.ImageRepository, + Version: vcp.Spec.PluginSources.Yaml.Version, + }) + if err != nil { + return fmt.Errorf("error when parsing virtual cluster plugin node-local-dns: %w", err) + } + pluginYamlContent = nodeLocalDNS + } + + pluginJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("plugin-executor-job-%s", vcp.Name), + Namespace: vcp.Namespace, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Name: "plugin-executor", + // ToDo Use unified image, requirements: contains kubectl & helm binary exec files + Image: vcp.Spec.PluginSources.Yaml.ImageRepository + "/kubectl:v1.25.7", + Command: []string{"sh", "-c"}, + Args: []string{ + fmt.Sprintf("echo '%s' | kubectl apply -f -", pluginYamlContent), + }, + }, + }, + }, + }, + }, + } + + _, err := vcpc.RootClientSet.BatchV1().Jobs(vcp.Namespace).Create(ctx, pluginJob, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("error when creating virtual cluster plugin %s, err: %w", pluginJob.Name, err) + } + + return nil +} + +func (vcpc *VirtualClusterPluginController) createPluginExecutorJobByHelm(ctx context.Context, vcp *v1alpha1.VirtualClusterPlugin) error { + pluginJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("plugin-executor-job-%s", vcp.Name), + Namespace: vcp.Namespace, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Name: "plugin-executor", + // ToDo Use unified image, requirements: contains kubectl & helm binary exec files + Image: "", + Command: []string{"sh", "-c"}, + Args: []string{ + fmt.Sprintf("helm install %s /mnt/helm-charts/%s", vcp.Name, vcp.Name), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin-charts", + MountPath: "/mnt/plugin-charts", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "plugin-charts", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + // local charts path + Path: vcp.Spec.PluginSources.Chart.Storage.HostPath.Path, + }, + }, + }, + }, + }, + }, + }, + } + + _, err := vcpc.RootClientSet.BatchV1().Jobs(vcp.Namespace).Create(ctx, pluginJob, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("error when creating virtual cluster plugin %s, err: %w", pluginJob.Name, err) + } + + return nil +} + +func (vcpc *VirtualClusterPluginController) SetupWithManager(mgr manager.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + Named(constants.PluginControllerName). + WithOptions(controller.Options{MaxConcurrentReconciles: 5}). + For(&v1alpha1.VirtualCluster{}, builder.WithPredicates(predicate.Funcs{ + CreateFunc: func(createEvent event.CreateEvent) bool { + return true + }, + UpdateFunc: func(updateEvent event.UpdateEvent) bool { + return true + }, + DeleteFunc: func(deleteEvent event.DeleteEvent) bool { + return true + }, + })). + Complete(vcpc) +} diff --git a/pkg/kubenest/manifest/plugins/manifest_node-local-dns.go b/pkg/kubenest/manifest/plugins/manifest_node-local-dns.go new file mode 100644 index 000000000..8b7af6277 --- /dev/null +++ b/pkg/kubenest/manifest/plugins/manifest_node-local-dns.go @@ -0,0 +1,227 @@ +package plugins + +const ( + NodeLocalDNSPlugin = ` +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-local-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns-upstream + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeDNSUpstream" +spec: + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-local-dns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + Corefile: | + {{ .Domain }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ .LocalDNS }} {{ .KubeDNS }} + forward . {{ .ClusterDNS }} { + force_tcp + } + prometheus :9253 + health {{ .LocalDNS }}:8080 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ .LocalDNS }} {{ .KubeDNS }} + forward . {{ .ClusterDNS }} { + force_tcp + } + prometheus :9253 + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ .LocalDNS }} {{ .KubeDNS }} + forward . {{ .ClusterDNS }} { + force_tcp + } + prometheus :9253 + } + .:53 { + errors + cache 30 + reload + loop + bind {{ .LocalDNS }} {{ .KubeDNS }} + forward . __PILLAR__UPSTREAM__SERVERS__ + prometheus :9253 + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-local-dns + namespace: kube-system + labels: + k8s-app: node-local-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + k8s-app: node-local-dns + template: + metadata: + labels: + k8s-app: node-local-dns + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: node-local-dns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + containers: + - name: node-cache + image: {{ .ImageRepository }}/k8s-dns-node-cache:{{ .Version }} + resources: + requests: + cpu: 25m + memory: 5Mi + args: [ "-localip", "{{ .LocalDNS }},{{ .KubeDNS }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + securityContext: + capabilities: + add: + - NET_ADMIN + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + host: {{ .LocalDNS }} + path: /health + port: 8080 + initialDelaySeconds: 60 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - name: config-volume + mountPath: /etc/coredns + - name: kube-dns-config + mountPath: /etc/kube-dns + volumes: + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + - name: config-volume + configMap: + name: node-local-dns + items: + - key: Corefile + path: Corefile.base +--- +# A headless service is a service with a service IP but instead of load-balancing it will return the IPs of our associated Pods. +# We use this to expose metrics to Prometheus. +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + labels: + k8s-app: node-local-dns + name: node-local-dns + namespace: kube-system +spec: + clusterIP: None + ports: + - name: metrics + port: 9253 + targetPort: 9253 + selector: + k8s-app: node-local-dns +` +) + +type NodeLocalDNSPluginReplace struct { + Domain string + KubeDNS string + LocalDNS string + ClusterDNS string + + Version string + ImageRepository string +}