From 29a33bddacdf0aa904f583400d01620e8054283f Mon Sep 17 00:00:00 2001 From: baoyinghai_yewu Date: Wed, 26 Jun 2024 15:18:12 +0800 Subject: [PATCH] feat: support ipv6 Signed-off-by: baoyinghai_yewu --- pkg/kubenest/constants/constant.go | 51 +++++++++++------- .../virtualcluster.node.controller/env/env.go | 9 ++++ .../exector/exector.go | 3 +- .../exector/remote_cmd.go | 18 ++++++- pkg/kubenest/controlplane/apiserver.go | 22 +++++--- pkg/kubenest/controlplane/component.go | 40 +++++++++----- pkg/kubenest/controlplane/etcd.go | 44 +++++++++------ .../apiserver/mainfests_deployment.go | 24 ++++++--- .../controlplane/etcd/mainfests_deployment.go | 22 ++++++-- .../kubecontroller/manifests_deployment.go | 10 ++-- .../scheduler/manifest_deployment.go | 4 +- pkg/kubenest/tasks/anp.go | 36 ++++++++----- pkg/kubenest/tasks/upload.go | 5 +- pkg/kubenest/util/cert/certs.go | 18 +++---- pkg/kubenest/util/image.go | 8 +++ pkg/kubenest/util/util.go | 53 ++++++++++++++----- pkg/utils/utils.go | 17 ++++++ 17 files changed, 272 insertions(+), 112 deletions(-) diff --git a/pkg/kubenest/constants/constant.go b/pkg/kubenest/constants/constant.go index a657a7c40..914b7c265 100644 --- a/pkg/kubenest/constants/constant.go +++ b/pkg/kubenest/constants/constant.go @@ -1,25 +1,30 @@ package constants -import "time" +import ( + "time" + + "github.com/kosmos.io/kosmos/pkg/utils" +) const ( - InitControllerName = "virtual-cluster-init-controller" - NodeControllerName = "virtual-cluster-node-controller" - GlobalNodeControllerName = "global-node-controller" - KosmosJoinControllerName = "kosmos-join-controller" - KosmosNs = "kosmos-system" - SystemNs = "kube-system" - DefauleImageRepositoryEnv = "IMAGE_REPOSITIRY" - DefauleImageVersionEnv = "IMAGE_VERSION" - VirtualClusterFinalizerName = "kosmos.io/virtual-cluster-finalizer" - ServiceType = "NodePort" - EtcdServiceType = "ClusterIP" - DisableCascadingDeletionLabel = "operator.virtualcluster.io/disable-cascading-deletion" - ControllerFinalizerName = "operator.virtualcluster.io/finalizer" - DefaultKubeconfigPath = "/etc/cluster-tree/cert" - Label = "virtualCluster-app" - ComponentBeReadyTimeout = 300 * time.Second - ComponentBeDeletedTimeout = 300 * time.Second + InitControllerName = "virtual-cluster-init-controller" + NodeControllerName = "virtual-cluster-node-controller" + GlobalNodeControllerName = "global-node-controller" + KosmosJoinControllerName = "kosmos-join-controller" + KosmosNs = "kosmos-system" + SystemNs = "kube-system" + DefauleImageRepositoryEnv = "IMAGE_REPOSITIRY" + DefauleImageVersionEnv = "IMAGE_VERSION" + DefauleVirtualControllerLabelEnv = "VIRTUAL_CONTROLLER_LABEL" + VirtualClusterFinalizerName = "kosmos.io/virtual-cluster-finalizer" + ServiceType = "NodePort" + EtcdServiceType = "ClusterIP" + DisableCascadingDeletionLabel = "operator.virtualcluster.io/disable-cascading-deletion" + ControllerFinalizerName = "operator.virtualcluster.io/finalizer" + DefaultKubeconfigPath = "/etc/cluster-tree/cert" + Label = "virtualCluster-app" + ComponentBeReadyTimeout = 300 * time.Second + ComponentBeDeletedTimeout = 300 * time.Second // CertificateBlockType is a possible value for pem.Block.Type. CertificateBlockType = "CERTIFICATE" @@ -41,7 +46,6 @@ const ( //controlplane apiserver ApiServer = "apiserver" ApiServerAnp = "apiserver-anp" - ApiServerServiceSubnet = "10.237.6.0/18" ApiServerEtcdListenClientPort = 2379 ApiServerServiceType = "NodePort" // APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation @@ -114,3 +118,12 @@ const ( ) type Action string + +var ApiServerServiceSubnet string +var KubeControllerManagerPodSubnet string + +func init() { + ApiServerServiceSubnet = utils.GetEnvWithDefaultValue("SERVICE_SUBNET", "10.237.6.0/18") + // fd11:1122:1111::/48, + KubeControllerManagerPodSubnet = utils.GetEnvWithDefaultValue("POD_SUBNET", "10.244.0.0/16") +} diff --git a/pkg/kubenest/controller/virtualcluster.node.controller/env/env.go b/pkg/kubenest/controller/virtualcluster.node.controller/env/env.go index 0956c38a5..10d3046d9 100644 --- a/pkg/kubenest/controller/virtualcluster.node.controller/env/env.go +++ b/pkg/kubenest/controller/virtualcluster.node.controller/env/env.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "strconv" + "strings" "k8s.io/klog" ) @@ -147,3 +148,11 @@ func GetNodeTaskMaxGoroutines() int { } return num } + +func GetCMDPaths() []string { + cmdAbsolutePaths := os.Getenv("CMD_ABSOLUTE_PATHS") + if len(cmdAbsolutePaths) == 0 { + return nil + } + return strings.Split(cmdAbsolutePaths, ",") +} diff --git a/pkg/kubenest/controller/virtualcluster.node.controller/exector/exector.go b/pkg/kubenest/controller/virtualcluster.node.controller/exector/exector.go index 306132a5b..e2e26a0b5 100644 --- a/pkg/kubenest/controller/virtualcluster.node.controller/exector/exector.go +++ b/pkg/kubenest/controller/virtualcluster.node.controller/exector/exector.go @@ -11,6 +11,7 @@ import ( "k8s.io/klog/v2" env "github.com/kosmos.io/kosmos/pkg/kubenest/controller/virtualcluster.node.controller/env" + "github.com/kosmos.io/kosmos/pkg/utils" ) type Status int @@ -175,6 +176,6 @@ func NewExectorHelper(addr string, port string) *ExectorHelper { token := env.GetExectorToken() return &ExectorHelper{ Token: token, - Addr: fmt.Sprintf("%s:%s", addr, exectorPort), + Addr: utils.GenerateAddrStr(addr, exectorPort), } } diff --git a/pkg/kubenest/controller/virtualcluster.node.controller/exector/remote_cmd.go b/pkg/kubenest/controller/virtualcluster.node.controller/exector/remote_cmd.go index a0d9a2dfc..98a9aed2b 100644 --- a/pkg/kubenest/controller/virtualcluster.node.controller/exector/remote_cmd.go +++ b/pkg/kubenest/controller/virtualcluster.node.controller/exector/remote_cmd.go @@ -1,19 +1,35 @@ package exector import ( + "fmt" "strings" "github.com/gorilla/websocket" + + env "github.com/kosmos.io/kosmos/pkg/kubenest/controller/virtualcluster.node.controller/env" ) type CMDExector struct { Cmd string } +func AddPrefix(cmd string) string { + cmdAbsolutePaths := env.GetCMDPaths() + if len(cmdAbsolutePaths) == 0 { + return cmd + } + for _, cmdAbsolutePath := range cmdAbsolutePaths { + if strings.HasSuffix(cmdAbsolutePath, fmt.Sprintf("/%s", cmd)) { + return cmdAbsolutePath + } + } + return cmd +} + func (e *CMDExector) GetWebSocketOption() WebSocketOption { cmdArgs := strings.Split(e.Cmd, " ") command := cmdArgs[0] - rawQuery := "command=" + command + rawQuery := "command=" + AddPrefix(command) if len(cmdArgs) > 1 { args := cmdArgs[1:] rawQuery = rawQuery + "&args=" + strings.Join(args, "&args=") diff --git a/pkg/kubenest/controlplane/apiserver.go b/pkg/kubenest/controlplane/apiserver.go index 56165bfea..c6c6f2074 100644 --- a/pkg/kubenest/controlplane/apiserver.go +++ b/pkg/kubenest/controlplane/apiserver.go @@ -36,18 +36,27 @@ func installAPIServer(client clientset.Interface, name, namespace string, portMa return nil } + vclabel := util.GetVirtualControllerLabel() + + IPV6FirstFlag, err := util.IPV6First(constants.ApiServerServiceSubnet) + if err != nil { + return err + } + apiserverDeploymentBytes, err := util.ParseTemplate(apiserver.ApiserverDeployment, struct { - DeploymentName, Namespace, ImageRepository, EtcdClientService, Version string - ServiceSubnet, VirtualClusterCertsSecret, EtcdCertsSecret string - Replicas int - EtcdListenClientPort int32 - ClusterPort int32 - AdmissionPlugins bool + DeploymentName, Namespace, ImageRepository, EtcdClientService, Version, VirtualControllerLabel string + ServiceSubnet, VirtualClusterCertsSecret, EtcdCertsSecret string + Replicas int + EtcdListenClientPort int32 + ClusterPort int32 + AdmissionPlugins bool + IPV6First bool }{ DeploymentName: fmt.Sprintf("%s-%s", name, "apiserver"), Namespace: namespace, ImageRepository: imageRepository, Version: imageVersion, + VirtualControllerLabel: vclabel, EtcdClientService: clusterIp, ServiceSubnet: constants.ApiServerServiceSubnet, VirtualClusterCertsSecret: fmt.Sprintf("%s-%s", name, "cert"), @@ -56,6 +65,7 @@ func installAPIServer(client clientset.Interface, name, namespace string, portMa EtcdListenClientPort: constants.ApiServerEtcdListenClientPort, ClusterPort: portMap[constants.ApiServerPortKey], AdmissionPlugins: opt.AdmissionPlugins, + IPV6First: IPV6FirstFlag, }) if err != nil { return fmt.Errorf("error when parsing virtual cluster apiserver deployment template: %w", err) diff --git a/pkg/kubenest/controlplane/component.go b/pkg/kubenest/controlplane/component.go index bba8acef3..dcb617e5f 100644 --- a/pkg/kubenest/controlplane/component.go +++ b/pkg/kubenest/controlplane/component.go @@ -113,19 +113,31 @@ func getComponentConfigmaps(component string) []string { func getKubeControllerManagerManifest(name, namespace string) (*appsv1.Deployment, error) { imageRepository, imageVersion := util.GetImageMessage() + + vclabel := util.GetVirtualControllerLabel() + + IPV6FirstFlag, err := util.IPV6First(constants.ApiServerServiceSubnet) + if err != nil { + return nil, err + } + kubeControllerManagerBytes, err := util.ParseTemplate(controller.KubeControllerManagerDeployment, struct { - DeploymentName, Namespace, ImageRepository, Version string - VirtualClusterCertsSecret, KubeconfigSecret, ServiceSubnet string - Replicas int32 + DeploymentName, Namespace, ImageRepository, Version, VirtualControllerLabel string + VirtualClusterCertsSecret, KubeconfigSecret, ServiceSubnet, PodSubnet string + Replicas int32 + IPV6First bool }{ DeploymentName: fmt.Sprintf("%s-%s", name, "kube-controller-manager"), Namespace: namespace, ImageRepository: imageRepository, Version: imageVersion, + VirtualControllerLabel: vclabel, VirtualClusterCertsSecret: fmt.Sprintf("%s-%s", name, "cert"), KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config-clusterip"), ServiceSubnet: constants.ApiServerServiceSubnet, + PodSubnet: constants.KubeControllerManagerPodSubnet, Replicas: constants.KubeControllerReplicas, + IPV6First: IPV6FirstFlag, }) if err != nil { return nil, fmt.Errorf("error when parsing kube-controller-manager deployment template: %w", err) @@ -160,18 +172,20 @@ func getVirtualClusterSchedulerConfigMapManifest(name, namespace string) (*v1.Co func getVirtualClusterSchedulerManifest(name, namespace string) (*appsv1.Deployment, error) { imageRepository, imageVersion := util.GetImageMessage() + vclabel := util.GetVirtualControllerLabel() virtualClusterSchedulerBytes, err := util.ParseTemplate(scheduler.VirtualClusterSchedulerDeployment, struct { - Replicas int32 - DeploymentName, Namespace, SystemNamespace, ImageRepository, Version string - Image, KubeconfigSecret string + Replicas int32 + DeploymentName, Namespace, SystemNamespace, ImageRepository, Version, VirtualControllerLabel string + Image, KubeconfigSecret string }{ - DeploymentName: fmt.Sprintf("%s-%s", name, "virtualcluster-scheduler"), - Namespace: namespace, - SystemNamespace: constants.SystemNs, - ImageRepository: imageRepository, - Version: imageVersion, - KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config-clusterip"), - Replicas: constants.VirtualClusterSchedulerReplicas, + DeploymentName: fmt.Sprintf("%s-%s", name, "virtualcluster-scheduler"), + Namespace: namespace, + SystemNamespace: constants.SystemNs, + ImageRepository: imageRepository, + VirtualControllerLabel: vclabel, + Version: imageVersion, + KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config-clusterip"), + Replicas: constants.VirtualClusterSchedulerReplicas, }) if err != nil { return nil, fmt.Errorf("error when parsing virtualCluster-scheduler deployment template: %w", err) diff --git a/pkg/kubenest/controlplane/etcd.go b/pkg/kubenest/controlplane/etcd.go index 2c9b0e9bd..349f2fea6 100644 --- a/pkg/kubenest/controlplane/etcd.go +++ b/pkg/kubenest/controlplane/etcd.go @@ -46,25 +46,35 @@ func installEtcd(client clientset.Interface, name, namespace string) error { initialClusters[index] = fmt.Sprintf("%s=%s", memberName, memberPeerURL) } + vclabel := util.GetVirtualControllerLabel() + + IPV6FirstFlag, err := util.IPV6First(constants.ApiServerServiceSubnet) + if err != nil { + return err + } + etcdStatefulSetBytes, err := util.ParseTemplate(etcd.EtcdStatefulSet, struct { - StatefulSetName, Namespace, ImageRepository, Image, EtcdClientService, Version string - CertsSecretName, EtcdPeerServiceName string - InitialCluster, EtcdDataVolumeName, EtcdCipherSuites string - Replicas, EtcdListenClientPort, EtcdListenPeerPort int32 + StatefulSetName, Namespace, ImageRepository, Image, EtcdClientService, Version, VirtualControllerLabel string + CertsSecretName, EtcdPeerServiceName string + InitialCluster, EtcdDataVolumeName, EtcdCipherSuites string + Replicas, EtcdListenClientPort, EtcdListenPeerPort int32 + IPV6First bool }{ - StatefulSetName: fmt.Sprintf("%s-%s", name, "etcd"), - Namespace: namespace, - ImageRepository: imageRepository, - Version: imageVersion, - EtcdClientService: fmt.Sprintf("%s-%s", name, "etcd-client"), - CertsSecretName: fmt.Sprintf("%s-%s", name, "etcd-cert"), - EtcdPeerServiceName: fmt.Sprintf("%s-%s", name, "etcd"), - EtcdDataVolumeName: constants.EtcdDataVolumeName, - InitialCluster: strings.Join(initialClusters, ","), - EtcdCipherSuites: strings.Join(flag.PreferredTLSCipherNames(), ","), - Replicas: constants.EtcdReplicas, - EtcdListenClientPort: constants.EtcdListenClientPort, - EtcdListenPeerPort: constants.EtcdListenPeerPort, + StatefulSetName: fmt.Sprintf("%s-%s", name, "etcd"), + Namespace: namespace, + ImageRepository: imageRepository, + Version: imageVersion, + VirtualControllerLabel: vclabel, + EtcdClientService: fmt.Sprintf("%s-%s", name, "etcd-client"), + CertsSecretName: fmt.Sprintf("%s-%s", name, "etcd-cert"), + EtcdPeerServiceName: fmt.Sprintf("%s-%s", name, "etcd"), + EtcdDataVolumeName: constants.EtcdDataVolumeName, + InitialCluster: strings.Join(initialClusters, ","), + EtcdCipherSuites: strings.Join(flag.PreferredTLSCipherNames(), ","), + Replicas: constants.EtcdReplicas, + EtcdListenClientPort: constants.EtcdListenClientPort, + EtcdListenPeerPort: constants.EtcdListenPeerPort, + IPV6First: IPV6FirstFlag, }) if err != nil { return fmt.Errorf("error when parsing Etcd statefuelset template: %w", err) diff --git a/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go b/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go index 18e732d29..348070322 100644 --- a/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go +++ b/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go @@ -24,7 +24,7 @@ spec: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet tolerations: - - key: "node-role.kubernetes.io/control-plane" + - key: {{ .VirtualControllerLabel }} operator: "Exists" effect: "NoSchedule" affinity: @@ -32,8 +32,8 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: {{ .VirtualControllerLabel }} + operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 @@ -66,8 +66,12 @@ spec: - --etcd-certfile=/etc/etcd/pki/etcd-client.crt - --etcd-keyfile=/etc/etcd/pki/etcd-client.key #- --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }} + {{ if .IPV6First }} + - --etcd-servers=https://[{{ .EtcdClientService }}]:{{ .EtcdListenClientPort }} + {{ else }} - --etcd-servers=https://{{ .EtcdClientService }}:{{ .EtcdListenClientPort }} - - --bind-address=0.0.0.0 + {{ end }} + - '--bind-address=::' - --kubelet-client-certificate=/etc/virtualcluster/pki/virtualCluster.crt - --kubelet-client-key=/etc/virtualcluster/pki/virtualCluster.key - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname @@ -160,7 +164,7 @@ spec: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet tolerations: - - key: "node-role.kubernetes.io/control-plane" + - key: {{ .VirtualControllerLabel }} operator: "Exists" effect: "NoSchedule" affinity: @@ -168,8 +172,8 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: {{ .VirtualControllerLabel }} + operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 @@ -202,8 +206,12 @@ spec: - --etcd-certfile=/etc/etcd/pki/etcd-client.crt - --etcd-keyfile=/etc/etcd/pki/etcd-client.key #- --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }} + {{ if .IPV6First }} + - --etcd-servers=https://[{{ .EtcdClientService }}]:{{ .EtcdListenClientPort }} + {{ else }} - --etcd-servers=https://{{ .EtcdClientService }}:{{ .EtcdListenClientPort }} - - --bind-address=0.0.0.0 + {{ end }} + - '--bind-address=::' - --kubelet-client-certificate=/etc/virtualcluster/pki/virtualCluster.crt - --kubelet-client-key=/etc/virtualcluster/pki/virtualCluster.key - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname diff --git a/pkg/kubenest/manifest/controlplane/etcd/mainfests_deployment.go b/pkg/kubenest/manifest/controlplane/etcd/mainfests_deployment.go index 35757fcb1..9f52a6092 100644 --- a/pkg/kubenest/manifest/controlplane/etcd/mainfests_deployment.go +++ b/pkg/kubenest/manifest/controlplane/etcd/mainfests_deployment.go @@ -25,7 +25,7 @@ spec: spec: automountServiceAccountToken: false tolerations: - - key: "node-role.kubernetes.io/control-plane" + - key: {{ .VirtualControllerLabel }} operator: "Exists" effect: "NoSchedule" affinity: @@ -33,7 +33,7 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: node-role.kubernetes.io/control-plane + - key: {{ .VirtualControllerLabel }} operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -53,10 +53,15 @@ spec: command: - /usr/local/bin/etcd - --name=$(VIRTUAL_ETCD_NAME) - - --listen-client-urls= https://0.0.0.0:{{ .EtcdListenClientPort }} - - --listen-peer-urls=http://0.0.0.0:{{ .EtcdListenPeerPort }} + - --listen-client-urls= https://[::]:{{ .EtcdListenClientPort }} + - --listen-peer-urls=http://[::]:{{ .EtcdListenPeerPort }} - --advertise-client-urls=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }} - --initial-cluster={{ .InitialCluster }} + {{ if .IPV6First }} + - --initial-advertise-peer-urls=http://[$(PODIP)]:2380 + {{ else }} + - --initial-advertise-peer-urls=http://$(PODIP):2380 + {{ end }} - --initial-cluster-state=new - --client-cert-auth=true - --trusted-ca-file=/etc/virtualcluster/pki/etcd/etcd-ca.crt @@ -71,6 +76,11 @@ spec: #- --peer-key-file=/etc/virtualcluster/pki/etcd/etcd-server.key #- --peer-trusted-ca-file=/etc/virtualcluster/pki/etcd/etcd-ca.crt env: + - name: PODIP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP - name: VIRTUAL_ETCD_NAME valueFrom: fieldRef: @@ -81,7 +91,11 @@ spec: command: - /bin/sh - -ec + {{ if .IPV6First }} + - etcdctl get /registry --prefix --keys-only --endpoints https://[::1]:{{ .EtcdListenClientPort }} --cacert=/etc/virtualcluster/pki/etcd/etcd-ca.crt --cert=/etc/virtualcluster/pki/etcd/etcd-server.crt --key=/etc/virtualcluster/pki/etcd/etcd-server.key + {{ else }} - etcdctl get /registry --prefix --keys-only --endpoints https://127.0.0.1:{{ .EtcdListenClientPort }} --cacert=/etc/virtualcluster/pki/etcd/etcd-ca.crt --cert=/etc/virtualcluster/pki/etcd/etcd-server.crt --key=/etc/virtualcluster/pki/etcd/etcd-server.key + {{ end }} failureThreshold: 3 initialDelaySeconds: 600 periodSeconds: 60 diff --git a/pkg/kubenest/manifest/controlplane/kubecontroller/manifests_deployment.go b/pkg/kubenest/manifest/controlplane/kubecontroller/manifests_deployment.go index ee863c737..f1d6447b6 100644 --- a/pkg/kubenest/manifest/controlplane/kubecontroller/manifests_deployment.go +++ b/pkg/kubenest/manifest/controlplane/kubecontroller/manifests_deployment.go @@ -24,7 +24,7 @@ spec: automountServiceAccountToken: false priorityClassName: system-node-critical tolerations: - - key: "node-role.kubernetes.io/control-plane" + - key: {{ .VirtualControllerLabel }} operator: "Exists" effect: "NoSchedule" affinity: @@ -32,7 +32,7 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: node-role.kubernetes.io/control-plane + - key: {{ .VirtualControllerLabel }} operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -55,15 +55,17 @@ spec: - --kubeconfig=/etc/virtualcluster/kubeconfig - --authentication-kubeconfig=/etc/virtualcluster/kubeconfig - --authorization-kubeconfig=/etc/virtualcluster/kubeconfig - - --bind-address=0.0.0.0 + - '--bind-address=::' - --client-ca-file=/etc/virtualcluster/pki/ca.crt - - --cluster-cidr=10.244.0.0/16 + - --cluster-cidr={{ .PodSubnet }} - --cluster-name=virtualcluster - --cluster-signing-cert-file=/etc/virtualcluster/pki/ca.crt - --cluster-signing-key-file=/etc/virtualcluster/pki/ca.key - --controllers=*,namespace,garbagecollector,serviceaccount-token,ttl-after-finished,bootstrapsigner,csrapproving,csrcleaner,csrsigning,clusterrole-aggregation - --leader-elect=true + {{ if not .IPV6First }} - --node-cidr-mask-size=24 + {{ end }} - --root-ca-file=/etc/virtualcluster/pki/ca.crt - --service-account-private-key-file=/etc/virtualcluster/pki/virtualCluster.key - --service-cluster-ip-range={{ .ServiceSubnet }} diff --git a/pkg/kubenest/manifest/controlplane/scheduler/manifest_deployment.go b/pkg/kubenest/manifest/controlplane/scheduler/manifest_deployment.go index 11843d5be..09b01703f 100644 --- a/pkg/kubenest/manifest/controlplane/scheduler/manifest_deployment.go +++ b/pkg/kubenest/manifest/controlplane/scheduler/manifest_deployment.go @@ -22,7 +22,7 @@ spec: spec: automountServiceAccountToken: false tolerations: - - key: "node-role.kubernetes.io/control-plane" + - key: {{ .VirtualControllerLabel }} operator: "Exists" effect: "NoSchedule" affinity: @@ -30,7 +30,7 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: node-role.kubernetes.io/control-plane + - key: {{ .VirtualControllerLabel }} operator: Exists podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: diff --git a/pkg/kubenest/tasks/anp.go b/pkg/kubenest/tasks/anp.go index 99f08864c..f943e0ee0 100644 --- a/pkg/kubenest/tasks/anp.go +++ b/pkg/kubenest/tasks/anp.go @@ -154,25 +154,34 @@ func installAnpServer(client clientset.Interface, name, namespace string, portMa return nil } + IPV6FirstFlag, err := util.IPV6First(constants.ApiServerServiceSubnet) + if err != nil { + return err + } + + vclabel := util.GetVirtualControllerLabel() + apiserverDeploymentBytes, err := util.ParseTemplate(apiserver.ApiserverAnpDeployment, struct { - DeploymentName, Namespace, ImageRepository, EtcdClientService, Version string - ServiceSubnet, VirtualClusterCertsSecret, EtcdCertsSecret string - Replicas int - EtcdListenClientPort int32 - ClusterPort int32 - AgentPort int32 - ServerPort int32 - HealthPort int32 - AdminPort int32 - KubeconfigSecret string - Name string - AnpMode string - AdmissionPlugins bool + DeploymentName, Namespace, ImageRepository, EtcdClientService, Version, VirtualControllerLabel string + ServiceSubnet, VirtualClusterCertsSecret, EtcdCertsSecret string + Replicas int + EtcdListenClientPort int32 + ClusterPort int32 + AgentPort int32 + ServerPort int32 + HealthPort int32 + AdminPort int32 + KubeconfigSecret string + Name string + AnpMode string + AdmissionPlugins bool + IPV6First bool }{ DeploymentName: fmt.Sprintf("%s-%s", name, "apiserver"), Namespace: namespace, ImageRepository: imageRepository, Version: imageVersion, + VirtualControllerLabel: vclabel, EtcdClientService: clusterIp, ServiceSubnet: constants.ApiServerServiceSubnet, VirtualClusterCertsSecret: fmt.Sprintf("%s-%s", name, "cert"), @@ -188,6 +197,7 @@ func installAnpServer(client clientset.Interface, name, namespace string, portMa Name: name, AnpMode: kubeNestOpt.AnpMode, AdmissionPlugins: kubeNestOpt.AdmissionPlugins, + IPV6First: IPV6FirstFlag, }) if err != nil { return fmt.Errorf("error when parsing virtual cluster apiserver deployment template: %w", err) diff --git a/pkg/kubenest/tasks/upload.go b/pkg/kubenest/tasks/upload.go index 53b7fa626..aa8c33a97 100644 --- a/pkg/kubenest/tasks/upload.go +++ b/pkg/kubenest/tasks/upload.go @@ -18,6 +18,7 @@ import ( "github.com/kosmos.io/kosmos/pkg/kubenest/util" "github.com/kosmos.io/kosmos/pkg/kubenest/util/cert" "github.com/kosmos.io/kosmos/pkg/kubenest/workflow" + "github.com/kosmos.io/kosmos/pkg/utils" ) var ( @@ -177,14 +178,14 @@ func runUploadAdminKubeconfig(r workflow.RunData) error { } portInfo := getPortInfoFromAPIServerService(service) // controlplane address + nodePort - controlplaneIpEndpoint = fmt.Sprintf("https://%s:%d", data.ControlplaneAddress(), portInfo.NodePort) + controlplaneIpEndpoint = fmt.Sprintf("https://%s", utils.GenerateAddrStr(data.ControlplaneAddress(), fmt.Sprintf("%d", portInfo.NodePort))) controlplaneIpKubeconfig, err := buildKubeConfigFromSpec(data, controlplaneIpEndpoint) if err != nil { return err } //clusterIP address + clusterIPPort - clusterIPEndpoint = fmt.Sprintf("https://%s:%d", service.Spec.ClusterIP, portInfo.ClusterIPPort) + clusterIPEndpoint = fmt.Sprintf("https://%s", utils.GenerateAddrStr(service.Spec.ClusterIP, fmt.Sprintf("%d", portInfo.ClusterIPPort))) clusterIPKubeconfig, err := buildKubeConfigFromSpec(data, clusterIPEndpoint) if err != nil { return err diff --git a/pkg/kubenest/util/cert/certs.go b/pkg/kubenest/util/cert/certs.go index dd4b19c57..908d29c4b 100644 --- a/pkg/kubenest/util/cert/certs.go +++ b/pkg/kubenest/util/cert/certs.go @@ -146,7 +146,7 @@ func etcdServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, altNames := &certutil.AltNames{ DNSNames: []string{"localhost", etcdClientServiceDNS, etcdPeerServiceDNS}, - IPs: []net.IP{net.IPv4(127, 0, 0, 1)}, + IPs: []net.IP{net.ParseIP("::1"), net.IPv4(127, 0, 0, 1)}, } if len(cfg.ClusterIps) > 0 { @@ -204,7 +204,7 @@ func makeAltNamesMutator(f func(cfg *AltNamesMutatorConfig) (*certutil.AltNames, } func proxyServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, error) { - firstIP, err := util.GetFirstIP(constants.ApiServerServiceSubnet) + firstIPs, err := util.GetFirstIP(constants.ApiServerServiceSubnet) if err != nil { return nil, err } @@ -216,10 +216,10 @@ func proxyServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, "kubernetes.default", "kubernetes.default.svc", }, - IPs: []net.IP{ + IPs: append([]net.IP{ + net.ParseIP("::1"), net.IPv4(127, 0, 0, 1), - firstIP, - }, + }, firstIPs...), } if cfg.Namespace != constants.VirtualClusterSystemNamespace { @@ -241,7 +241,7 @@ func proxyServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, } func apiServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, error) { - firstIP, err := util.GetFirstIP(constants.ApiServerServiceSubnet) + firstIPs, err := util.GetFirstIP(constants.ApiServerServiceSubnet) if err != nil { return nil, err } @@ -257,10 +257,10 @@ func apiServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, e fmt.Sprintf("*.%s.svc", constants.VirtualClusterSystemNamespace), }, //TODO （考虑节点属于当前集群节点和非当前集群节点情况） - IPs: []net.IP{ + IPs: append([]net.IP{ + net.ParseIP("::1"), net.IPv4(127, 0, 0, 1), - firstIP, - }, + }, firstIPs...), } if cfg.Namespace != constants.VirtualClusterSystemNamespace { diff --git a/pkg/kubenest/util/image.go b/pkg/kubenest/util/image.go index bfea9d98e..b3f93cff7 100644 --- a/pkg/kubenest/util/image.go +++ b/pkg/kubenest/util/image.go @@ -18,3 +18,11 @@ func GetImageMessage() (imageRepository string, imageVersion string) { } return imageRepository, imageVersion } + +func GetVirtualControllerLabel() string { + lb := os.Getenv(constants.DefauleVirtualControllerLabelEnv) + if len(lb) == 0 { + return "node-role.kubernetes.io/control-plane" + } + return lb +} diff --git a/pkg/kubenest/util/util.go b/pkg/kubenest/util/util.go index 1e40c4500..dacf2fa30 100644 --- a/pkg/kubenest/util/util.go +++ b/pkg/kubenest/util/util.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "fmt" "net" + "strings" "k8s.io/client-go/kubernetes" @@ -42,22 +43,48 @@ func GenerateKubeclient(virtualCluster *v1alpha1.VirtualCluster) (kubernetes.Int return k8sClient, nil } -func GetFirstIP(ipNetStr string) (net.IP, error) { - ip, ipNet, err := net.ParseCIDR(ipNetStr) - if err != nil { - return nil, fmt.Errorf("parse ipNetStr failed: %s", err) +func GetFirstIP(ipNetStrs string) ([]net.IP, error) { + ipNetStrArray := strings.Split(ipNetStrs, ",") + if len(ipNetStrArray) > 2 { + return nil, fmt.Errorf("getFirstIP failed, ipstring is too long: %s", ipNetStrs) } - ip = ip.To4() - if ip == nil { - return nil, fmt.Errorf("only support ipv4") - } + var ips []net.IP + for _, ipNetStr := range ipNetStrArray { + ip, ipNet, err := net.ParseCIDR(ipNetStr) + if err != nil { + return nil, fmt.Errorf("parse ipNetStr failed: %s", err) + } - networkIP := ip.Mask(ipNet.Mask) + networkIP := ip.Mask(ipNet.Mask) - firstIP := make(net.IP, len(networkIP)) - copy(firstIP, networkIP) - firstIP[3]++ + // IPv4 + if ip.To4() != nil { + firstIP := make(net.IP, len(networkIP)) + copy(firstIP, networkIP) + firstIP[len(firstIP)-1]++ + ips = append(ips, firstIP) + continue + } - return firstIP, nil + // IPv6 + firstIP := make(net.IP, len(networkIP)) + copy(firstIP, networkIP) + for i := len(firstIP) - 1; i >= 0; i-- { + firstIP[i]++ + if firstIP[i] != 0 { + break + } + } + ips = append(ips, firstIP) + } + return ips, nil +} + +func IPV6First(ipNetStr string) (bool, error) { + ipNetStrArray := strings.Split(ipNetStr, ",") + if len(ipNetStrArray) > 2 { + return false, fmt.Errorf("getFirstIP failed, ipstring is too long: %s", ipNetStr) + } + return utils.IsIPv6(ipNetStrArray[0]), nil } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 28e89b7ff..544f153fd 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -1,6 +1,8 @@ package utils import ( + "fmt" + "os" "strings" ) @@ -25,3 +27,18 @@ func IsIPv6(s string) bool { } return false } + +func GetEnvWithDefaultValue(envName string, defaultValue string) string { + v := os.Getenv(envName) + if len(v) == 0 { + return defaultValue + } + return v +} + +func GenerateAddrStr(addr string, port string) string { + if IsIPv6(addr) { + return fmt.Sprintf("[%s]:%s", addr, port) + } + return fmt.Sprintf("%s:%s", addr, port) +}