fix: add kube-proxy in virtualcluster

Signed-off-by: renxiangyu <[email protected]>

pkg/kubenest/constants/constant.go

@@ -48,6 +48,11 @@ const (
 	ApiServerCallRetryInterval = 100 * time.Millisecond
 	APIServerSVCPortName       = "client"
 
+	//install kube-proxy in virtualCluster
+	Proxy = "kube-proxy"
+	// configmap kube-proxy clustercidr
+	ProxyClusterClusterCIDR = "172.19.0.0/16,fd22:2222:2222::/48"
+
 	//controlplane etcd
 	Etcd                 = "etcd"
 	EtcdReplicas         = 3

pkg/kubenest/controlplane/proxy.go

@@ -0,0 +1,126 @@
+package controlplane
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	clientset "k8s.io/client-go/kubernetes"
+
+	"github.com/kosmos.io/kosmos/pkg/kubenest/constants"
+	"github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/proxy"
+	"github.com/kosmos.io/kosmos/pkg/kubenest/util"
+)
+
+func EnsureVirtualClusterProxy(client clientset.Interface, kubeconfigString string) error {
+	// install kube-proxy ds in virtual cluster
+	if err := installProxyDaemonSet(client); err != nil {
+		return fmt.Errorf("failed to install virtual cluster proxy, err: %w", err)
+	}
+
+	// install kube-proxy cm in virtual cluster
+	if err := installProxyConfigMap(client, kubeconfigString); err != nil {
+		return fmt.Errorf("failed to install virtual cluster proxy, err: %w", err)
+	}
+
+	// install kube-proxy sa in virtual cluster
+	if err := installProxySA(client); err != nil {
+		return fmt.Errorf("failed to install virtual cluster proxy, err: %w", err)
+	}
+	return nil
+}
+
+func DeleteVirtualClusterProxy(client clientset.Interface) error {
+	daemonSetName := fmt.Sprintf("%s-%s", "kube", "proxy")
+	daemonSetNameSpace := fmt.Sprintf("%s-%s", "kube", "system")
+	if err := util.DeleteDaemonSet(client, daemonSetName, daemonSetNameSpace); err != nil {
+		return errors.Wrapf(err, "Failed to delete daemonSet %s/%s", daemonSetName, daemonSetNameSpace)
+	}
+
+	cmName := fmt.Sprintf("%s-%s", "kube", "proxy")
+	cmNameSpace := fmt.Sprintf("%s-%s", "kube", "system")
+	if err := util.DeleteConfigmap(client, cmName, cmNameSpace); err != nil {
+		return errors.Wrapf(err, "Failed to delete ConfigMap %s/%s", cmName, cmNameSpace)
+	}
+
+	saName := fmt.Sprintf("%s-%s", "kube", "proxy")
+	saNameSpace := fmt.Sprintf("%s-%s", "kube", "system")
+	if err := util.DeleteServiceAccount(client, saName, saNameSpace); err != nil {
+		return errors.Wrapf(err, "Failed to delete ServiceAccount %s/%s", saName, saNameSpace)
+	}
+	return nil
+}
+
+func installProxyDaemonSet(client clientset.Interface) error {
+	imageRepository, imageVersion := util.GetImageMessage()
+
+	proxyDaemonSetBytes, err := util.ParseTemplate(proxy.ProxyDaemonSet, struct {
+		DaemonSetName, Namespace, ImageRepository, Version string
+	}{
+		DaemonSetName:   fmt.Sprintf("%s-%s", "kube", "proxy"),
+		Namespace:       fmt.Sprintf("%s-%s", "kube", "system"),
+		ImageRepository: imageRepository,
+		Version:         imageVersion,
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing virtual cluster proxy daemonSet template: %w", err)
+	}
+
+	proxyDaemonSet := &appsv1.DaemonSet{}
+	if err := yaml.Unmarshal([]byte(proxyDaemonSetBytes), proxyDaemonSet); err != nil {
+		return fmt.Errorf("error when decoding virtual cluster proxy daemonSet: %w", err)
+	}
+
+	if err := util.CreateOrUpdateDaemonSet(client, proxyDaemonSet); err != nil {
+		return fmt.Errorf("error when creating daemonSet for %s, err: %w", proxyDaemonSet.Name, err)
+	}
+	return nil
+}
+
+func installProxyConfigMap(client clientset.Interface, kubeconfigString string) error {
+	proxyConfigMapBytes, err := util.ParseTemplate(proxy.ProxyConfigMap, struct {
+		ConfigMapName, Namespace, KubeProxyKubeConfig, ClusterCIDR string
+	}{
+		ConfigMapName:       fmt.Sprintf("%s-%s", "kube", "proxy"),
+		Namespace:           fmt.Sprintf("%s-%s", "kube", "system"),
+		KubeProxyKubeConfig: kubeconfigString,
+		ClusterCIDR:         constants.ProxyClusterClusterCIDR,
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing virtual cluster proxy configmap template: %w", err)
+	}
+
+	proxyConfigMap := &corev1.ConfigMap{}
+	if err := yaml.Unmarshal([]byte(proxyConfigMapBytes), proxyConfigMap); err != nil {
+		return fmt.Errorf("error when decoding virtual cluster proxy configmap: %w", err)
+	}
+
+	if err := util.CreateOrUpdateConfigMap(client, proxyConfigMap); err != nil {
+		return fmt.Errorf("error when creating configmap for %s, err: %w", proxyConfigMap.Name, err)
+	}
+	return nil
+}
+
+func installProxySA(client clientset.Interface) error {
+	proxySABytes, err := util.ParseTemplate(proxy.ProxySA, struct {
+		SAName, Namespace string
+	}{
+		SAName:    fmt.Sprintf("%s-%s", "kube", "proxy"),
+		Namespace: fmt.Sprintf("%s-%s", "kube", "system"),
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing virtual cluster proxy SA template: %w", err)
+	}
+
+	proxySA := &corev1.ServiceAccount{}
+	if err := yaml.Unmarshal([]byte(proxySABytes), proxySA); err != nil {
+		return fmt.Errorf("error when decoding virtual cluster proxy SA: %w", err)
+	}
+
+	if err := util.CreateOrUpdateServiceAccount(client, proxySA); err != nil {
+		return fmt.Errorf("error when creating SA for %s, err: %w", proxySA.Name, err)
+	}
+	return nil
+}

pkg/kubenest/init.go

@@ -63,6 +63,8 @@ func NewInitPhase(opts *InitOptions) *workflow.Phase {
 	initPhase.AppendTask(tasks.NewAnpTask())
 	initPhase.AppendTask(tasks.NewComponentTask())
 	initPhase.AppendTask(tasks.NewCheckControlPlaneTask())
+	// create proxy
+	initPhase.AppendTask(tasks.NewVirtualClusterProxyTask())
 	// create core-dns
 	initPhase.AppendTask(tasks.NewCoreDNSTask())
 	// add server
@@ -84,6 +86,7 @@ func UninstallPhase(opts *InitOptions) *workflow.Phase {
 	destroyPhase.AppendTask(tasks.UninstallVirtualClusterServiceTask())
 	destroyPhase.AppendTask(tasks.UninstallCertsAndKubeconfigTask())
 	destroyPhase.AppendTask(tasks.DeleteEtcdPvcTask())
+	destroyPhase.AppendTask(tasks.UninstallVirtualClusterProxyTask())
 
 	destroyPhase.SetDataInitializer(func() (workflow.RunData, error) {
 		return newRunData(opts)

pkg/kubenest/manifest/controlplane/proxy/mainfests_daemonset.go

@@ -0,0 +1,147 @@
+package proxy
+
+const (
+	ProxyDaemonSet = `
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ .DaemonSetName }}
+  namespace: {{ .Namespace }}
+  labels:
+    virtualCluster-app: kube-proxy
+    app.kubernetes.io/managed-by: virtual-cluster-controller
+spec:
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app.kubernetes.io/managed-by: virtual-cluster-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/managed-by: virtual-cluster-controller
+    spec:
+      containers:
+      - command:
+        - /usr/local/bin/kube-proxy
+        - --config=/var/lib/kube-proxy/config.conf
+        - --hostname-override=$(NODE_NAME)
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        image: {{ .ImageRepository }}/kube-proxy:{{ .Version }}
+        imagePullPolicy: IfNotPresent
+        name: kube-proxy
+        resources: {}
+        securityContext:
+          privileged: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /var/lib/kube-proxy
+          name: kube-proxy
+        - mountPath: /run/xtables.lock
+          name: xtables-lock
+        - mountPath: /lib/modules
+          name: lib-modules
+          readOnly: true
+      dnsPolicy: ClusterFirst
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-node-critical
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      serviceAccount: kube-proxy
+      serviceAccountName: kube-proxy
+      terminationGracePeriodSeconds: 30
+      tolerations:
+      - operator: Exists
+      volumes:
+      - configMap:
+          defaultMode: 420
+          name: kube-proxy
+        name: kube-proxy
+      - hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+        name: xtables-lock
+      - hostPath:
+          path: /lib/modules
+          type: ""
+        name: lib-modules
+  updateStrategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
+`
+	ProxyConfigMap = `
+apiVersion: v1
+data:
+  config.conf: |-
+    apiVersion: kubeproxy.config.k8s.io/v1alpha1
+    bindAddress: 0.0.0.0
+    bindAddressHardFail: false
+    clientConnection:
+      acceptContentTypes: ""
+      burst: 100
+      contentType: ""
+      kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
+      qps: 100
+    clusterCIDR: {{ .ClusterCIDR }}
+    configSyncPeriod: 0s
+    conntrack:
+      maxPerCore: null
+      min: null
+      tcpCloseWaitTimeout: null
+      tcpEstablishedTimeout: null
+    detectLocal:
+      bridgeInterface: ""
+      interfaceNamePrefix: ""
+    detectLocalMode: ""
+    enableProfiling: false
+    healthzBindAddress: ""
+    hostnameOverride: ""
+    iptables:
+      masqueradeAll: true
+      masqueradeBit: null
+      minSyncPeriod: 0s
+      syncPeriod: 0s
+    ipvs:
+      excludeCIDRs:
+      - 192.0.0.1/32
+      minSyncPeriod: 0s
+      scheduler: ""
+      strictARP: false
+      syncPeriod: 0s
+      tcpFinTimeout: 0s
+      tcpTimeout: 0s
+      udpTimeout: 0s
+    kind: KubeProxyConfiguration
+    metricsBindAddress: 0.0.0.0:10249
+    mode: ipvs
+    nodePortAddresses: null
+    oomScoreAdj: null
+    portRange: ""
+    showHiddenMetricsForVersion: ""
+    udpIdleTimeout: 0s
+    winkernel:
+      enableDSR: false
+      forwardHealthCheckVip: false
+      networkName: ""
+      rootHnsEndpointName: ""
+      sourceVip: ""
+  kubeconfig.conf: |-
+    {{ .KubeProxyKubeConfig }}
+kind: ConfigMap
+metadata:
+  labels:
+    app: kube-proxy
+  name: {{ .ConfigMapName }}
+  namespace: {{ .Namespace }}
+`
+)

pkg/kubenest/manifest/controlplane/proxy/manifests_rbac.go

@@ -0,0 +1,11 @@
+package proxy
+
+const (
+	ProxySA = `
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .SAName }}
+  namespace: {{ .Namespace }}
+`
+)