feat: add pluggable component installation method

Signed-off-by: ONE7live <[email protected]>

charts/plugins/node-local-dns/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/plugins/node-local-dns/Chart.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1alpha1
+name: node-local-dns
+namespace: kube-system
+description: A Helm chart for Kubenest plugin Node-local-dns
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+version: 0.0.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application.
+appVersion: latest

charts/plugins/node-local-dns/README.md

@@ -0,0 +1,33 @@
+# Node-Local-DNS
+
+Kosmos-kubenest plugin NodeLocalDNS helm chart
+
+## Summary
+
+The chart install NodeLocalDNS set according to <https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/>.
+
+It is designed to work both with Iptables and IPVS setup.
+
+Latest available `node-local-dns` image can be found at [node-local-dns google container repository](https://console.cloud.google.com/gcr/images/google-containers/GLOBAL/k8s-dns-node-cache)
+
+## Values
+
+| Key                       | Type | Default                                    | Description |
+|---------------------------|------|--------------------------------------------|-------------|
+| image.repository          | string | `"registry.k8s.io/dns/k8s-dns-node-cache"` |  |
+| image.version             | string | `"1.23.1"`                                 |  |
+| image.pullPolicy          | string | `"IfNotPresent"`                           |  |
+| config.domain             | string | `"cluster.local"`                          |  |
+| config.kubeDNS            | string | `"xxx.xxx.xxx.xxx"`                        |  |
+| config.localDNS           | string | `"xxx.xxx.xxx.xxx"`                        |  |
+| config.clusterDNS         | string | `"xxx.xxx.xxx.xxx"`                        |  |
+| resources.requests.cpu    | string | `"25m"`                                    |  |
+| resources.requests.memory | string | `"5Mi"`                                    |  |
+| tolerations[0].key        | string | `"CriticalAddonsOnly"`                     |  |
+| tolerations[0].operator   | string | `"Exists"`                                 |  |
+| tolerations[1].effect     | string | `"NoExecute"`                              |  |
+| tolerations[1].operator   | string | `"Exists"`                                 |  |
+| tolerations[2].effect     | string | `"NoSchedule"`                             |  |
+| tolerations[2].operator   | string | `"Exists"`                                 |  |
+| nodeSelector              | object | `{}`                                       |  |
+| affinity                  | object | `{}`                                       |  |

charts/plugins/node-local-dns/templates/_helpers.tpl

@@ -0,0 +1,7 @@
+{{- define "node-local-dns.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{- define "node-local-dns.namespace" -}}
+{{- default .Chart.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}

charts/plugins/node-local-dns/templates/configmap.yaml

@@ -0,0 +1,56 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "node-local-dns.name" . }}
+  namespace: {{ include "node-local-dns.namespace" . }}
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+data:
+  Corefile: |
+    {{ .Values.configMap.domain }}:53 {
+        errors
+        cache {
+                success 9984 30
+                denial 9984 5
+        }
+        reload
+        loop
+        bind {{ .Values.configMap.localDNS }} {{ .Values.configMap.kubeDNS }}
+        forward . {{ .Values.configMap.clusterDNS }} {
+                force_tcp
+        }
+        prometheus :9253
+        health {{ .Values.configMap.localDNS }}:8080
+        }
+    in-addr.arpa:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ .Values.configMap.localDNS }} {{ .Values.configMap.kubeDNS }}
+        forward . {{ .Values.configMap.clusterDNS }} {
+                force_tcp
+        }
+        prometheus :9253
+        }
+    ip6.arpa:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ .Values.configMap.localDNS }} {{ .Values.configMap.kubeDNS }}
+        forward . {{ .Values.configMap.clusterDNS }} {
+                force_tcp
+        }
+        prometheus :9253
+        }
+    .:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ .Values.configMap.localDNS }} {{ .Values.configMap.kubeDNS }}
+        forward . __PILLAR__UPSTREAM__SERVERS__
+        prometheus :9253
+        }

charts/plugins/node-local-dns/templates/daemonset.yaml

@@ -0,0 +1,82 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ include "node-local-dns.name" . }}
+  namespace: {{ include "node-local-dns.namespace" . }}
+  labels:
+    k8s-app: {{ include "node-local-dns.name" . }}
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 10%
+  selector:
+    matchLabels:
+      k8s-app: {{ include "node-local-dns.name" . }}
+  template:
+    metadata:
+      labels:
+        k8s-app: {{ include "node-local-dns.name" . }}
+      annotations:
+        prometheus.io/port: "9253"
+        prometheus.io/scrape: "true"
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: {{ include "node-local-dns.name" . }}
+      hostNetwork: true
+      dnsPolicy: Default  # Don't use cluster DNS.
+      tolerations:
+        {{- toYaml .Values.tolerations | nindent 8 }}
+      containers:
+        - name: node-cache
+          image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.version }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          args: [ "-localip", "{{ .Values.configMap.localDNS }},{{ .Values.configMap.kubeDNS }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ]
+          securityContext:
+            capabilities:
+              add:
+                - NET_ADMIN
+          ports:
+            - containerPort: 53
+              name: dns
+              protocol: UDP
+            - containerPort: 53
+              name: dns-tcp
+              protocol: TCP
+            - containerPort: 9253
+              name: metrics
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              host: {{ .Values.configMap.localDNS }}
+              path: /health
+              port: 8080
+            initialDelaySeconds: 60
+            timeoutSeconds: 5
+          volumeMounts:
+            - mountPath: /run/xtables.lock
+              name: xtables-lock
+              readOnly: false
+            - name: config-volume
+              mountPath: /etc/coredns
+            - name: kube-dns-config
+              mountPath: /etc/kube-dns
+      volumes:
+        - name: xtables-lock
+          hostPath:
+            path: /run/xtables.lock
+            type: FileOrCreate
+        - name: kube-dns-config
+          configMap:
+            name: kube-dns
+            optional: true
+        - name: config-volume
+          configMap:
+            name: {{ include "node-local-dns.name" . }}
+            items:
+              - key: Corefile
+                path: Corefile.base

charts/plugins/node-local-dns/templates/service.yaml

@@ -0,0 +1,44 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-dns-upstream
+  namespace: {{ include "node-local-dns.namespace" . }}
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/name: "KubeDNSUpstream"
+spec:
+  ports:
+    - name: dns
+      port: 53
+      protocol: UDP
+      targetPort: 53
+    - name: dns-tcp
+      port: 53
+      protocol: TCP
+      targetPort: 53
+  selector:
+    k8s-app: kube-dns
+---
+# A headless service is a service with a service IP but instead of load-balancing it will return the IPs of our associated Pods.
+# We use this to expose metrics to Prometheus.
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    prometheus.io/port: "9253"
+    prometheus.io/scrape: "true"
+  labels:
+    k8s-app: {{ include "node-local-dns.name" . }}
+  name: {{ include "node-local-dns.name" . }}
+  namespace: {{ include "node-local-dns.namespace" . }}
+spec:
+  clusterIP: None
+  ports:
+    - name: metrics
+      port: 9253
+      targetPort: 9253
+  selector:
+    k8s-app: {{ include "node-local-dns.name" . }}

charts/plugins/node-local-dns/templates/serviceaccount.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "node-local-dns.name" . }}
+  namespace: {{ include "node-local-dns.namespace" . }}
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile

charts/plugins/node-local-dns/values.yaml

@@ -0,0 +1,27 @@
+---
+image:
+    repository: registry.k8s.io/dns/k8s-dns-node-cache
+    version: 1.23.1
+    pullPolicy: IfNotPresent
+---
+config:
+  domain: xxx.xxx
+  kubeDNS: xxx.xxx.xxx
+  localDNS: xxx.xxx.xxx
+  clusterDNS: xxx.xxx.xxx
+---
+tolerations:
+  - key: CriticalAddonsOnly
+    operator: Exists
+  - effect: NoExecute
+    operator: Exists
+  - effect: NoSchedule
+    operator: Exists
+---
+resources:
+  requests:
+    cpu: 25m
+    memory: 5Mi
+---
+nodeSelector: {}
+affinity: {}

cmd/kubenest/operator/app/operator.go

@@ -260,7 +260,6 @@ func run(ctx context.Context, config *config.Config) error {
 		KosmosClient:  kosmosClient,
 		EventRecorder: mgr.GetEventRecorderFor(constants.GlobalNodeControllerName),
 	}
-
 	if err = GlobalNodeController.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error starting %s: %v", constants.GlobalNodeControllerName, err)
 	}
@@ -294,6 +293,16 @@ func run(ctx context.Context, config *config.Config) error {
 		}
 	}
 
+	//VirtualClusterPluginController := vcpc.VirtualClusterPluginController{
+	//	Client:        mgr.GetClient(),
+	//	RootClientSet: hostKubeClient,
+	//	KosmosClient:  kosmosClient,
+	//	EventRecorder: mgr.GetEventRecorderFor(constants.PluginControllerName),
+	//}
+	//if err = VirtualClusterPluginController.SetupWithManager(mgr); err != nil {
+	//	return fmt.Errorf("error starting %s: %v", constants.PluginControllerName, err)
+	//}
+
 	if err := mgr.Start(ctx); err != nil {
 		return fmt.Errorf("failed to start controller manager: %v", err)
 	}

pkg/apis/kosmos/v1alpha1/virtualclusterplugin_types.go

@@ -51,6 +51,24 @@ type Chart struct {
 type Yaml struct {
 	// +required
 	Path Storage `json:"path"`
+
+	// +optional
+	Domain string `json:"domain"`
+
+	// +optional
+	KubeDNS string `json:"kubeDNS"`
+
+	// +optional
+	LocalDNS string `json:"localDNS"`
+
+	// +optional
+	ClusterDNS string `json:"clusterDNS"`
+
+	// +optional
+	ImageRepository string `json:"imageRepository"`
+
+	// +optional
+	Version string `json:"version"`
 }
 
 type Storage struct {

pkg/kubenest/constants/constant.go

@@ -9,6 +9,7 @@ import (
 const (
 	InitControllerName               = "virtual-cluster-init-controller"
 	NodeControllerName               = "virtual-cluster-node-controller"
+	PluginControllerName             = "virtual-cluster-plugin-controller"
 	GlobalNodeControllerName         = "global-node-controller"
 	KosmosJoinControllerName         = "kosmos-join-controller"
 	KosmosNs                         = "kosmos-system"