Skip to content

Commit

Permalink
feat: support ipv6
Browse files Browse the repository at this point in the history 
Signed-off-by: baoyinghai_yewu <[email protected]>
  • Loading branch information
@OrangeBao
OrangeBao committed Jun 28, 2024
1 parent 032ceaf commit 4ee7aa2
Show file tree
Hide file tree
Showing 17 changed files with 282 additions and 118 deletions.
55 changes: 34 additions & 21 deletions pkg/kubenest/constants/constant.go
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,32 @@
package constants

import "time"
import (
"time"

"github.com/kosmos.io/kosmos/pkg/utils"
)

const (
InitControllerName = "virtual-cluster-init-controller"
NodeControllerName = "virtual-cluster-node-controller"
GlobalNodeControllerName = "global-node-controller"
KosmosJoinControllerName = "kosmos-join-controller"
KosmosNs = "kosmos-system"
SystemNs = "kube-system"
DefaultNs = "default"
DefaultImageRepositoryEnv = "IMAGE_REPOSITIRY"
DefaultImageVersionEnv = "IMAGE_VERSION"
DefaultCoreDnsImageTagEnv = "COREDNS_IMAGE_TAG"
VirtualClusterFinalizerName = "kosmos.io/virtual-cluster-finalizer"
ServiceType = "NodePort"
EtcdServiceType = "ClusterIP"
DisableCascadingDeletionLabel = "operator.virtualcluster.io/disable-cascading-deletion"
ControllerFinalizerName = "operator.virtualcluster.io/finalizer"
DefaultKubeconfigPath = "/etc/cluster-tree/cert"
Label = "virtualCluster-app"
ComponentBeReadyTimeout = 300 * time.Second
ComponentBeDeletedTimeout = 300 * time.Second
InitControllerName = "virtual-cluster-init-controller"
NodeControllerName = "virtual-cluster-node-controller"
GlobalNodeControllerName = "global-node-controller"
KosmosJoinControllerName = "kosmos-join-controller"
KosmosNs = "kosmos-system"
SystemNs = "kube-system"
DefaultNs = "default"
DefaultImageRepositoryEnv = "IMAGE_REPOSITIRY"
DefaultImageVersionEnv = "IMAGE_VERSION"
DefaultCoreDnsImageTagEnv = "COREDNS_IMAGE_TAG"
VirtualClusterFinalizerName = "kosmos.io/virtual-cluster-finalizer"
ServiceType = "NodePort"
EtcdServiceType = "ClusterIP"
DisableCascadingDeletionLabel = "operator.virtualcluster.io/disable-cascading-deletion"
ControllerFinalizerName = "operator.virtualcluster.io/finalizer"
DefaultKubeconfigPath = "/etc/cluster-tree/cert"
Label = "virtualCluster-app"
ComponentBeReadyTimeout = 300 * time.Second
ComponentBeDeletedTimeout = 300 * time.Second
DefauleVirtualControllerLabelEnv = "VIRTUAL_CONTROLLER_LABEL"

// CertificateBlockType is a possible value for pem.Block.Type.
CertificateBlockType = "CERTIFICATE"
Expand All @@ -43,7 +48,6 @@ const (
//controlplane apiserver
ApiServer = "apiserver"
ApiServerAnp = "apiserver-anp"
ApiServerServiceSubnet = "10.237.6.0/18"
ApiServerEtcdListenClientPort = 2379
ApiServerServiceType = "NodePort"
// APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation
Expand Down Expand Up @@ -121,3 +125,12 @@ const (
)

type Action string

var ApiServerServiceSubnet string
var KubeControllerManagerPodSubnet string

func init() {
ApiServerServiceSubnet = utils.GetEnvWithDefaultValue("SERVICE_SUBNET", "10.237.6.0/18")
// fd11:1122:1111::/48,
KubeControllerManagerPodSubnet = utils.GetEnvWithDefaultValue("POD_SUBNET", "10.244.0.0/16")
}
9 changes: 9 additions & 0 deletions pkg/kubenest/controller/virtualcluster.node.controller/env/env.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"os"
"strconv"
"strings"

"k8s.io/klog"
)
Expand Down Expand Up @@ -147,3 +148,11 @@ func GetNodeTaskMaxGoroutines() int {
}
return num
}

func GetCMDPaths() []string {
cmdAbsolutePaths := os.Getenv("CMD_ABSOLUTE_PATHS")
if len(cmdAbsolutePaths) == 0 {
return nil
}
return strings.Split(cmdAbsolutePaths, ",")
}
3 changes: 2 additions & 1 deletion pkg/kubenest/controller/virtualcluster.node.controller/exector/exector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"k8s.io/klog/v2"

env "github.com/kosmos.io/kosmos/pkg/kubenest/controller/virtualcluster.node.controller/env"
"github.com/kosmos.io/kosmos/pkg/utils"
)

type Status int
Expand Down Expand Up @@ -175,6 +176,6 @@ func NewExectorHelper(addr string, port string) *ExectorHelper {
token := env.GetExectorToken()
return &ExectorHelper{
Token: token,
Addr: fmt.Sprintf("%s:%s", addr, exectorPort),
Addr: utils.GenerateAddrStr(addr, exectorPort),
}
}
18 changes: 17 additions & 1 deletion pkg/kubenest/controller/virtualcluster.node.controller/exector/remote_cmd.go
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,35 @@
package exector

import (
"fmt"
"strings"

"github.com/gorilla/websocket"

env "github.com/kosmos.io/kosmos/pkg/kubenest/controller/virtualcluster.node.controller/env"
)

type CMDExector struct {
Cmd string
}

func AddPrefix(cmd string) string {
cmdAbsolutePaths := env.GetCMDPaths()
if len(cmdAbsolutePaths) == 0 {
return cmd
}
for _, cmdAbsolutePath := range cmdAbsolutePaths {
if strings.HasSuffix(cmdAbsolutePath, fmt.Sprintf("/%s", cmd)) {
return cmdAbsolutePath
}
}
return cmd
}

func (e *CMDExector) GetWebSocketOption() WebSocketOption {
cmdArgs := strings.Split(e.Cmd, " ")
command := cmdArgs[0]
rawQuery := "command=" + command
rawQuery := "command=" + AddPrefix(command)
if len(cmdArgs) > 1 {
args := cmdArgs[1:]
rawQuery = rawQuery + "&args=" + strings.Join(args, "&args=")
Expand Down
22 changes: 16 additions & 6 deletions pkg/kubenest/controlplane/apiserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,18 +36,27 @@ func installAPIServer(client clientset.Interface, name, namespace string, portMa
return nil
}

vclabel := util.GetVirtualControllerLabel()

IPV6FirstFlag, err := util.IPV6First(constants.ApiServerServiceSubnet)
if err != nil {
return err
}

apiserverDeploymentBytes, err := util.ParseTemplate(apiserver.ApiserverDeployment, struct {
DeploymentName, Namespace, ImageRepository, EtcdClientService, Version string
ServiceSubnet, VirtualClusterCertsSecret, EtcdCertsSecret string
Replicas int
EtcdListenClientPort int32
ClusterPort int32
AdmissionPlugins bool
DeploymentName, Namespace, ImageRepository, EtcdClientService, Version, VirtualControllerLabel string
ServiceSubnet, VirtualClusterCertsSecret, EtcdCertsSecret string
Replicas int
EtcdListenClientPort int32
ClusterPort int32
AdmissionPlugins bool
IPV6First bool
}{
DeploymentName: fmt.Sprintf("%s-%s", name, "apiserver"),
Namespace: namespace,
ImageRepository: imageRepository,
Version: imageVersion,
VirtualControllerLabel: vclabel,
EtcdClientService: clusterIp,
ServiceSubnet: constants.ApiServerServiceSubnet,
VirtualClusterCertsSecret: fmt.Sprintf("%s-%s", name, "cert"),
Expand All @@ -56,6 +65,7 @@ func installAPIServer(client clientset.Interface, name, namespace string, portMa
EtcdListenClientPort: constants.ApiServerEtcdListenClientPort,
ClusterPort: portMap[constants.ApiServerPortKey],
AdmissionPlugins: opt.AdmissionPlugins,
IPV6First: IPV6FirstFlag,
})
if err != nil {
return fmt.Errorf("error when parsing virtual cluster apiserver deployment template: %w", err)
Expand Down
46 changes: 32 additions & 14 deletions pkg/kubenest/controlplane/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,20 +113,36 @@ func getComponentConfigmaps(component string) []string {

func getKubeControllerManagerManifest(name, namespace, clusterCIDR string) (*appsv1.Deployment, error) {
imageRepository, imageVersion := util.GetImageMessage()

vclabel := util.GetVirtualControllerLabel()

IPV6FirstFlag, err := util.IPV6First(constants.ApiServerServiceSubnet)
if err != nil {
return nil, err
}

podSubnet := constants.KubeControllerManagerPodSubnet
if len(clusterCIDR) > 0 {
podSubnet = clusterCIDR
}

kubeControllerManagerBytes, err := util.ParseTemplate(controller.KubeControllerManagerDeployment, struct {
DeploymentName, Namespace, ImageRepository, Version, ClusterCIDR string
VirtualClusterCertsSecret, KubeconfigSecret, ServiceSubnet string
Replicas int32
DeploymentName, Namespace, ImageRepository, Version, VirtualControllerLabel, PodSubnet string
VirtualClusterCertsSecret, KubeconfigSecret, ServiceSubnet string
Replicas int32
IPV6First bool
}{
DeploymentName: fmt.Sprintf("%s-%s", name, "kube-controller-manager"),
Namespace: namespace,
ImageRepository: imageRepository,
Version: imageVersion,
VirtualControllerLabel: vclabel,
VirtualClusterCertsSecret: fmt.Sprintf("%s-%s", name, "cert"),
KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config-clusterip"),
ServiceSubnet: constants.ApiServerServiceSubnet,
PodSubnet: podSubnet,
Replicas: constants.KubeControllerReplicas,
ClusterCIDR: clusterCIDR,
IPV6First: IPV6FirstFlag,
})
if err != nil {
return nil, fmt.Errorf("error when parsing kube-controller-manager deployment template: %w", err)
Expand Down Expand Up @@ -161,18 +177,20 @@ func getVirtualClusterSchedulerConfigMapManifest(name, namespace string) (*v1.Co

func getVirtualClusterSchedulerManifest(name, namespace string) (*appsv1.Deployment, error) {
imageRepository, imageVersion := util.GetImageMessage()
vclabel := util.GetVirtualControllerLabel()
virtualClusterSchedulerBytes, err := util.ParseTemplate(scheduler.VirtualClusterSchedulerDeployment, struct {
Replicas int32
DeploymentName, Namespace, SystemNamespace, ImageRepository, Version string
Image, KubeconfigSecret string
Replicas int32
DeploymentName, Namespace, SystemNamespace, ImageRepository, Version, VirtualControllerLabel string
Image, KubeconfigSecret string
}{
DeploymentName: fmt.Sprintf("%s-%s", name, "virtualcluster-scheduler"),
Namespace: namespace,
SystemNamespace: constants.SystemNs,
ImageRepository: imageRepository,
Version: imageVersion,
KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config-clusterip"),
Replicas: constants.VirtualClusterSchedulerReplicas,
DeploymentName: fmt.Sprintf("%s-%s", name, "virtualcluster-scheduler"),
Namespace: namespace,
SystemNamespace: constants.SystemNs,
ImageRepository: imageRepository,
VirtualControllerLabel: vclabel,
Version: imageVersion,
KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config-clusterip"),
Replicas: constants.VirtualClusterSchedulerReplicas,
})
if err != nil {
return nil, fmt.Errorf("error when parsing virtualCluster-scheduler deployment template: %w", err)
Expand Down
50 changes: 30 additions & 20 deletions pkg/kubenest/controlplane/etcd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,28 +59,38 @@ func installEtcd(client clientset.Interface, name, namespace string, ko *ko.Kube
initialClusters[index] = fmt.Sprintf("%s=%s", memberName, memberPeerURL)
}

vclabel := util.GetVirtualControllerLabel()

IPV6FirstFlag, err := util.IPV6First(constants.ApiServerServiceSubnet)
if err != nil {
return err
}

etcdStatefulSetBytes, err := util.ParseTemplate(etcd.EtcdStatefulSet, struct {
StatefulSetName, Namespace, ImageRepository, Image, EtcdClientService, Version string
CertsSecretName, EtcdPeerServiceName string
InitialCluster, EtcdDataVolumeName, EtcdCipherSuites string
Replicas, EtcdListenClientPort, EtcdListenPeerPort int32
ETCDStorageClass, ETCDStorageSize string
StatefulSetName, Namespace, ImageRepository, Image, EtcdClientService, Version, VirtualControllerLabel string
CertsSecretName, EtcdPeerServiceName string
InitialCluster, EtcdDataVolumeName, EtcdCipherSuites string
Replicas, EtcdListenClientPort, EtcdListenPeerPort int32
ETCDStorageClass, ETCDStorageSize string
IPV6First bool
}{
StatefulSetName: fmt.Sprintf("%s-%s", name, "etcd"),
Namespace: namespace,
ImageRepository: imageRepository,
Version: imageVersion,
EtcdClientService: fmt.Sprintf("%s-%s", name, "etcd-client"),
CertsSecretName: fmt.Sprintf("%s-%s", name, "etcd-cert"),
EtcdPeerServiceName: fmt.Sprintf("%s-%s", name, "etcd"),
EtcdDataVolumeName: constants.EtcdDataVolumeName,
InitialCluster: strings.Join(initialClusters, ","),
EtcdCipherSuites: strings.Join(flag.PreferredTLSCipherNames(), ","),
Replicas: constants.EtcdReplicas,
EtcdListenClientPort: constants.EtcdListenClientPort,
EtcdListenPeerPort: constants.EtcdListenPeerPort,
ETCDStorageClass: ko.ETCDStorageClass,
ETCDStorageSize: resourceQuantity.String(),
StatefulSetName: fmt.Sprintf("%s-%s", name, "etcd"),
Namespace: namespace,
ImageRepository: imageRepository,
Version: imageVersion,
VirtualControllerLabel: vclabel,
EtcdClientService: fmt.Sprintf("%s-%s", name, "etcd-client"),
CertsSecretName: fmt.Sprintf("%s-%s", name, "etcd-cert"),
EtcdPeerServiceName: fmt.Sprintf("%s-%s", name, "etcd"),
EtcdDataVolumeName: constants.EtcdDataVolumeName,
InitialCluster: strings.Join(initialClusters, ","),
EtcdCipherSuites: strings.Join(flag.PreferredTLSCipherNames(), ","),
Replicas: constants.EtcdReplicas,
EtcdListenClientPort: constants.EtcdListenClientPort,
EtcdListenPeerPort: constants.EtcdListenPeerPort,
ETCDStorageClass: ko.ETCDStorageClass,
ETCDStorageSize: resourceQuantity.String(),
IPV6First: IPV6FirstFlag,
})
if err != nil {
return fmt.Errorf("error when parsing Etcd statefuelset template: %w", err)
Expand Down
24 changes: 16 additions & 8 deletions pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,16 @@ spec:
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
tolerations:
- key: "node-role.kubernetes.io/control-plane"
- key: {{ .VirtualControllerLabel }}
operator: "Exists"
effect: "NoSchedule"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: Exists
- key: {{ .VirtualControllerLabel }}
operator: Exists
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
Expand Down Expand Up @@ -66,8 +66,12 @@ spec:
- --etcd-certfile=/etc/etcd/pki/etcd-client.crt
- --etcd-keyfile=/etc/etcd/pki/etcd-client.key
#- --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
{{ if .IPV6First }}
- --etcd-servers=https://[{{ .EtcdClientService }}]:{{ .EtcdListenClientPort }}
{{ else }}
- --etcd-servers=https://{{ .EtcdClientService }}:{{ .EtcdListenClientPort }}
- --bind-address=0.0.0.0
{{ end }}
- '--bind-address=::'
- --kubelet-client-certificate=/etc/virtualcluster/pki/virtualCluster.crt
- --kubelet-client-key=/etc/virtualcluster/pki/virtualCluster.key
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
Expand Down Expand Up @@ -160,16 +164,16 @@ spec:
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
tolerations:
- key: "node-role.kubernetes.io/control-plane"
- key: {{ .VirtualControllerLabel }}
operator: "Exists"
effect: "NoSchedule"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: Exists
- key: {{ .VirtualControllerLabel }}
operator: Exists
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
Expand Down Expand Up @@ -202,8 +206,12 @@ spec:
- --etcd-certfile=/etc/etcd/pki/etcd-client.crt
- --etcd-keyfile=/etc/etcd/pki/etcd-client.key
#- --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
{{ if .IPV6First }}
- --etcd-servers=https://[{{ .EtcdClientService }}]:{{ .EtcdListenClientPort }}
{{ else }}
- --etcd-servers=https://{{ .EtcdClientService }}:{{ .EtcdListenClientPort }}
- --bind-address=0.0.0.0
{{ end }}
- '--bind-address=::'
- --kubelet-client-certificate=/etc/virtualcluster/pki/virtualCluster.crt
- --kubelet-client-key=/etc/virtualcluster/pki/virtualCluster.key
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
Expand Down
Loading

0 comments on commit 4ee7aa2

Please sign in to comment.