From 35d464cdb855185182bcdb64217fa81dc5af4bf9 Mon Sep 17 00:00:00 2001 From: lyzuiui <164340845@qq.com> Date: Mon, 19 Aug 2024 16:27:31 +0800 Subject: [PATCH 1/3] issues-652-support fixed nodePort for virtualcluster apiserver Signed-off-by: lyzuiui <164340845@qq.com> --- .../kosmos/v1alpha1/virtualcluster_types.go | 8 +++++++ .../kosmos/v1alpha1/zz_generated.deepcopy.go | 5 ++++ pkg/generated/openapi/zz_generated.openapi.go | 22 +++++++++++++++++ .../virtualcluster_init_controller.go | 24 +++++++++++++++++++ 4 files changed, 59 insertions(+) diff --git a/pkg/apis/kosmos/v1alpha1/virtualcluster_types.go b/pkg/apis/kosmos/v1alpha1/virtualcluster_types.go index a91735ecc..956235b4c 100644 --- a/pkg/apis/kosmos/v1alpha1/virtualcluster_types.go +++ b/pkg/apis/kosmos/v1alpha1/virtualcluster_types.go @@ -54,6 +54,14 @@ type VirtualClusterSpec struct { // +optional ExternalIP string `json:"externalIP,omitempty"` + // ExternalPort is the port number for the external IP of the virtual kubernetes's control plane + // +optional + ExternalPort int32 `json:"externalPort,omitempty"` + + // ExternalIps is the external ips of the virtual kubernetes's control plane + // +optional + ExternalIps []string `json:"externalIps,omitempty"` + // PromotePolicies definites the policies for promote to the kubernetes's control plane // +required PromotePolicies []PromotePolicy `json:"promotePolicies,omitempty"` diff --git a/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go index f65fafbf1..1ed23db65 100644 --- a/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go @@ -1933,6 +1933,11 @@ func (in *VirtualClusterPluginSpec) DeepCopy() *VirtualClusterPluginSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualClusterSpec) DeepCopyInto(out *VirtualClusterSpec) { *out = *in + if in.ExternalIps != nil { + in, out := &in.ExternalIps, &out.ExternalIps + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.PromotePolicies != nil { in, out := &in.PromotePolicies, &out.PromotePolicies *out = make([]PromotePolicy, len(*in)) diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 9eab67a6e..0e7ad34e5 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -3367,6 +3367,28 @@ func schema_pkg_apis_kosmos_v1alpha1_VirtualClusterSpec(ref common.ReferenceCall Format: "", }, }, + "externalPort": { + SchemaProps: spec.SchemaProps{ + Description: "ExternalPort is the port number for the external IP of the virtual kubernetes's control plane", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "externalIps": { + SchemaProps: spec.SchemaProps{ + Description: "ExternalIps is the external ips of the virtual kubernetes's control plane", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, "promotePolicies": { SchemaProps: spec.SchemaProps{ Description: "PromotePolicies definites the policies for promote to the kubernetes's control plane", diff --git a/pkg/kubenest/controller/virtualcluster_init_controller.go b/pkg/kubenest/controller/virtualcluster_init_controller.go index 29c160ac1..9ac0d36c9 100644 --- a/pkg/kubenest/controller/virtualcluster_init_controller.go +++ b/pkg/kubenest/controller/virtualcluster_init_controller.go @@ -747,18 +747,37 @@ func (c *VirtualClusterInitController) AllocateHostPort(virtualCluster *v1alpha1 if len(virtualCluster.Status.PortMap) > 0 || virtualCluster.Status.Port != 0 { return 0, nil } + // 获取主机端口池 hostPool, err := GetHostPortPoolFromConfigMap(c.RootClientSet, constants.KosmosNs, constants.HostPortsCMName, constants.HostPortsCMDataName) if err != nil { return 0, err } + isPortInPool := func(port int32) bool { + for _, p := range hostPool.PortsPool { + if p == port { + return true + } + } + return false + } + + if virtualCluster.Spec.ExternalPort != 0 && !isPortInPool(virtualCluster.Spec.ExternalPort) { + return 0, fmt.Errorf("ExternalPort is not in host pool") + } hostAddress, err := c.findHostAddresses() if err != nil { return 0, err } + // 准备端口分配列表 ports := func() []int32 { ports := make([]int32, 0) + if virtualCluster.Spec.ExternalPort != 0 && !c.isPortAllocated(virtualCluster.Spec.ExternalPort, hostAddress) { + ports = append(ports, virtualCluster.Spec.ExternalPort) + } else if virtualCluster.Spec.ExternalPort != 0 && c.isPortAllocated(virtualCluster.Spec.ExternalPort, hostAddress) { + return nil + } for _, p := range hostPool.PortsPool { if !c.isPortAllocated(p, hostAddress) { ports = append(ports, p) @@ -769,9 +788,14 @@ func (c *VirtualClusterInitController) AllocateHostPort(virtualCluster *v1alpha1 } return ports }() + if ports == nil { + return 0, fmt.Errorf("port is already being used") + } + //可分配端口不够 if len(ports) < constants.VirtualClusterPortNum { return 0, fmt.Errorf("no available ports to allocate") } + // 分配端口并更新 PortMap virtualCluster.Status.PortMap = make(map[string]int32) virtualCluster.Status.PortMap[constants.ApiServerPortKey] = ports[0] virtualCluster.Status.PortMap[constants.ApiServerNetworkProxyAgentPortKey] = ports[1] From 4707b6b217768d01c9ecccfbd2fa07b2d7835ab2 Mon Sep 17 00:00:00 2001 From: lyzuiui <164340845@qq.com> Date: Fri, 23 Aug 2024 09:37:01 +0800 Subject: [PATCH 2/3] support fixed nodePort for virtualcluster apiserver Signed-off-by: lyzuiui <164340845@qq.com> --- .../v1alpha1/kubenestconfiguration_types.go | 2 ++ .../kosmos/v1alpha1/zz_generated.deepcopy.go | 3 --- pkg/generated/openapi/zz_generated.openapi.go | 19 ++++++------------- .../virtualcluster_init_controller.go | 11 ++++++----- .../virtualcluster_init_controller_test.go | 10 +++++----- 5 files changed, 19 insertions(+), 26 deletions(-) diff --git a/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go b/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go index ab339432f..62b727b55 100644 --- a/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go +++ b/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go @@ -51,6 +51,8 @@ type KubeInKubeConfig struct { ETCDStorageClass string `yaml:"etcdStorageClass" json:"etcdStorageClass,omitempty"` // +optional ETCDUnitSize string `yaml:"etcdUnitSize" json:"etcdUnitSize,omitempty"` + // +optional + ExternalPort int32 `json:"externalPort,omitempty"` //// Etcd contains the configuration for the etcd statefulset. //Etcd EtcdCluster `yaml:"etcd" json:"etcd,omitempty"` diff --git a/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go index b10a862d1..fe5679c60 100644 --- a/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go @@ -1965,10 +1965,7 @@ func (in *VirtualClusterSpec) DeepCopyInto(out *VirtualClusterSpec) { *out = make([]string, len(*in)) copy(*out, *in) } -<<<<<<< HEAD -======= in.KubeInKubeConfig.DeepCopyInto(&out.KubeInKubeConfig) ->>>>>>> upstream/main if in.PromotePolicies != nil { in, out := &in.PromotePolicies, &out.PromotePolicies *out = make([]PromotePolicy, len(*in)) diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 4a6e6e51c..022d326f0 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -1970,6 +1970,12 @@ func schema_pkg_apis_kosmos_v1alpha1_KubeInKubeConfig(ref common.ReferenceCallba Format: "", }, }, + "externalPort": { + SchemaProps: spec.SchemaProps{ + Type: []string{"integer"}, + Format: "int32", + }, + }, "tenantEntrypoint": { SchemaProps: spec.SchemaProps{ Description: "TenantEntrypoint TenantEntrypoint `yaml:\"tenantEntrypoint\" json:\"tenantEntrypoint,omitempty\"`", @@ -3421,16 +3427,6 @@ func schema_pkg_apis_kosmos_v1alpha1_VirtualClusterSpec(ref common.ReferenceCall Format: "", }, }, -<<<<<<< HEAD - "externalPort": { - SchemaProps: spec.SchemaProps{ - Description: "ExternalPort is the port number for the external IP of the virtual kubernetes's control plane", - Type: []string{"integer"}, - Format: "int32", - }, - }, -======= ->>>>>>> upstream/main "externalIps": { SchemaProps: spec.SchemaProps{ Description: "ExternalIps is the external ips of the virtual kubernetes's control plane", @@ -3446,8 +3442,6 @@ func schema_pkg_apis_kosmos_v1alpha1_VirtualClusterSpec(ref common.ReferenceCall }, }, }, -<<<<<<< HEAD -======= "kubeInKubeConfig": { SchemaProps: spec.SchemaProps{ Description: "KubeInKubeConfig is the external config of virtual cluster", @@ -3455,7 +3449,6 @@ func schema_pkg_apis_kosmos_v1alpha1_VirtualClusterSpec(ref common.ReferenceCall Ref: ref("github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.KubeInKubeConfig"), }, }, ->>>>>>> upstream/main "promotePolicies": { SchemaProps: spec.SchemaProps{ Description: "PromotePolicies definites the policies for promote to the kubernetes's control plane", diff --git a/pkg/kubenest/controller/virtualcluster_init_controller.go b/pkg/kubenest/controller/virtualcluster_init_controller.go index e168f35dc..dda0badba 100644 --- a/pkg/kubenest/controller/virtualcluster_init_controller.go +++ b/pkg/kubenest/controller/virtualcluster_init_controller.go @@ -4,11 +4,12 @@ import ( "context" "encoding/base64" "fmt" - "github.com/kosmos.io/kosmos/pkg/kubenest/tasks" "sort" "sync" "time" + "github.com/kosmos.io/kosmos/pkg/kubenest/tasks" + "github.com/pkg/errors" "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" @@ -828,7 +829,7 @@ func (c *VirtualClusterInitController) AllocateHostPort(virtualCluster *v1alpha1 return false } - if virtualCluster.Spec.ExternalPort != 0 && !isPortInPool(virtualCluster.Spec.ExternalPort) { + if virtualCluster.Spec.KubeInKubeConfig.ExternalPort != 0 && !isPortInPool(virtualCluster.Spec.KubeInKubeConfig.ExternalPort) { return 0, fmt.Errorf("ExternalPort is not in host pool") } @@ -840,9 +841,9 @@ func (c *VirtualClusterInitController) AllocateHostPort(virtualCluster *v1alpha1 // 准备端口分配列表 ports := func() []int32 { ports := make([]int32, 0) - if virtualCluster.Spec.ExternalPort != 0 && !c.isPortAllocated(virtualCluster.Spec.ExternalPort, hostAddress) { - ports = append(ports, virtualCluster.Spec.ExternalPort) - } else if virtualCluster.Spec.ExternalPort != 0 && c.isPortAllocated(virtualCluster.Spec.ExternalPort, hostAddress) { + if virtualCluster.Spec.KubeInKubeConfig.ExternalPort != 0 && !c.isPortAllocated(virtualCluster.Spec.KubeInKubeConfig.ExternalPort, hostAddress) { + ports = append(ports, virtualCluster.Spec.KubeInKubeConfig.ExternalPort) + } else if virtualCluster.Spec.KubeInKubeConfig.ExternalPort != 0 && c.isPortAllocated(virtualCluster.Spec.KubeInKubeConfig.ExternalPort, hostAddress) { return nil } for _, p := range hostPool.PortsPool { diff --git a/pkg/kubenest/controller/virtualcluster_init_controller_test.go b/pkg/kubenest/controller/virtualcluster_init_controller_test.go index 8e0fbeb5b..291b80355 100644 --- a/pkg/kubenest/controller/virtualcluster_init_controller_test.go +++ b/pkg/kubenest/controller/virtualcluster_init_controller_test.go @@ -46,15 +46,15 @@ func AllocateHostPortTemplate(virtualCluster *v1alpha1.VirtualCluster, usedPorts // 准备端口分配列表 //判断ExternalPort是否在主机端口池里面 - if virtualCluster.Spec.ExternalPort != 0 && !isPortInPool(virtualCluster.Spec.ExternalPort) { + if virtualCluster.Spec.KubeInKubeConfig.ExternalPort != 0 && !isPortInPool(virtualCluster.Spec.KubeInKubeConfig.ExternalPort) { return nil, nil } ports := func() []int32 { ports := make([]int32, 0) - if virtualCluster.Spec.ExternalPort != 0 && !isPortAllocated(virtualCluster.Spec.ExternalPort, usedPorts) { - ports = append(ports, virtualCluster.Spec.ExternalPort) - } else if isPortAllocated(virtualCluster.Spec.ExternalPort, usedPorts) { + if virtualCluster.Spec.KubeInKubeConfig.ExternalPort != 0 && !isPortAllocated(virtualCluster.Spec.KubeInKubeConfig.ExternalPort, usedPorts) { + ports = append(ports, virtualCluster.Spec.KubeInKubeConfig.ExternalPort) + } else if isPortAllocated(virtualCluster.Spec.KubeInKubeConfig.ExternalPort, usedPorts) { return nil } for _, p := range hostPoolPorts { @@ -172,7 +172,7 @@ func TestVirtualClusterInitController_AllocateHostPort(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { virtualClusterTest := &v1alpha1.VirtualCluster{} - virtualClusterTest.Spec.ExternalPort = tt.args.externalPort + virtualClusterTest.Spec.KubeInKubeConfig.ExternalPort = tt.args.externalPort got, _ := AllocateHostPortTemplate(virtualClusterTest, tt.args.usedPorts, tt.args.ports...) // if (err != nil) != tt.wantErr { // t.Errorf("VirtualClusterInitController.AllocateHostPort() error = %v, wantErr %v", err, tt.wantErr) From 81a270a7ad32384bccb7422461a3c9b8d621ff62 Mon Sep 17 00:00:00 2001 From: luoyuanze <164340845@qq.com> Date: Fri, 23 Aug 2024 17:42:58 +0800 Subject: [PATCH 3/3] avoid the virtual cluster apiserver restart we deploy anp Signed-off-by: luoyuanze <164340845@qq.com> --- .../apiserver/mainfests_deployment.go | 127 +++--------------- pkg/kubenest/tasks/anp.go | 3 +- 2 files changed, 23 insertions(+), 107 deletions(-) diff --git a/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go b/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go index 348070322..f0a97579b 100644 --- a/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go +++ b/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go @@ -32,8 +32,10 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: {{ .VirtualControllerLabel }} - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - kubenest-control-plane podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 @@ -142,7 +144,7 @@ apiVersion: apps/v1 kind: Deployment metadata: labels: - virtualCluster-app: apiserver + #virtualCluster-app: apiserver virtualCluster-anp: apiserver-anp app.kubernetes.io/managed-by: virtual-cluster-controller name: {{ .DeploymentName }} @@ -153,11 +155,11 @@ spec: type: Recreate selector: matchLabels: - virtualCluster-app: apiserver + virtualCluster-anp: apiserver-anp template: metadata: labels: - virtualCluster-app: apiserver + #virtualCluster-app: apiserver virtualCluster-anp: apiserver-anp spec: automountServiceAccountToken: false @@ -172,109 +174,24 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: {{ .VirtualControllerLabel }} - operator: Exists - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: virtualCluster-app + # - key: {{ .VirtualControllerLabel }} + # operator: Exists + - key: kubernetes.io/hostname operator: In values: - - apiserver - topologyKey: kubernetes.io/hostname + - kubenest-control-plane + #podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - weight: 100 + # podAffinityTerm: + # labelSelector: + # matchExpressions: + # - key: virtualCluster-app + # operator: In + # values: + # - apiserver + # topologyKey: kubernetes.io/hostname containers: - - name: kube-apiserver - image: {{ .ImageRepository }}/kube-apiserver:{{ .Version }} - imagePullPolicy: IfNotPresent - env: - - name: PODIP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - command: - - kube-apiserver - - --allow-privileged=true - - --authorization-mode=Node,RBAC - - --client-ca-file=/etc/virtualcluster/pki/ca.crt - - --enable-admission-plugins=NodeRestriction - - --enable-bootstrap-token-auth=true - - --etcd-cafile=/etc/etcd/pki/etcd-ca.crt - - --etcd-certfile=/etc/etcd/pki/etcd-client.crt - - --etcd-keyfile=/etc/etcd/pki/etcd-client.key - #- --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }} - {{ if .IPV6First }} - - --etcd-servers=https://[{{ .EtcdClientService }}]:{{ .EtcdListenClientPort }} - {{ else }} - - --etcd-servers=https://{{ .EtcdClientService }}:{{ .EtcdListenClientPort }} - {{ end }} - - '--bind-address=::' - - --kubelet-client-certificate=/etc/virtualcluster/pki/virtualCluster.crt - - --kubelet-client-key=/etc/virtualcluster/pki/virtualCluster.key - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --secure-port={{ .ClusterPort }} - - --service-account-issuer=https://kubernetes.default.svc.cluster.local - - --service-account-key-file=/etc/virtualcluster/pki/virtualCluster.key - - --service-account-signing-key-file=/etc/virtualcluster/pki/virtualCluster.key - - --service-cluster-ip-range={{ .ServiceSubnet }} - - --proxy-client-cert-file=/etc/virtualcluster/pki/front-proxy-client.crt - - --proxy-client-key-file=/etc/virtualcluster/pki/front-proxy-client.key - - --requestheader-allowed-names=front-proxy-client - - --requestheader-client-ca-file=/etc/virtualcluster/pki/front-proxy-ca.crt - - --requestheader-extra-headers-prefix=X-Remote-Extra- - - --requestheader-group-headers=X-Remote-Group - - --requestheader-username-headers=X-Remote-User - - --tls-cert-file=/etc/virtualcluster/pki/apiserver.crt - - --tls-private-key-file=/etc/virtualcluster/pki/apiserver.key - - --tls-min-version=VersionTLS13 - - --max-requests-inflight=1500 - - --max-mutating-requests-inflight=500 - - --v=4 - - --advertise-address=$(PODIP) - - --egress-selector-config-file=/etc/kubernetes/konnectivity-server-config/{{ .Namespace }}/{{ .Name }}/egress_selector_configuration.yaml - {{ if not .AdmissionPlugins }} - - --disable-admission-plugins=License - {{ end }} - livenessProbe: - failureThreshold: 8 - httpGet: - path: /livez - port: {{ .ClusterPort }} - scheme: HTTPS - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 15 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readyz - port: {{ .ClusterPort }} - scheme: HTTPS - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 15 - ports: - - containerPort: {{ .ClusterPort }} - name: http - protocol: TCP - volumeMounts: - - mountPath: /etc/virtualcluster/pki - name: apiserver-cert - readOnly: true - - mountPath: /etc/etcd/pki - name: etcd-cert - readOnly: true - - mountPath: /etc/kubernetes/konnectivity-server/{{ .Namespace }}/{{ .Name }} - readOnly: false - name: konnectivity-uds - - name: kas-proxy - mountPath: /etc/kubernetes/konnectivity-server-config/{{ .Namespace }}/{{ .Name }}/egress_selector_configuration.yaml - subPath: egress_selector_configuration.yaml - name: konnectivity-server-container image: {{ .ImageRepository }}/kas-network-proxy-server:{{ .Version }} resources: diff --git a/pkg/kubenest/tasks/anp.go b/pkg/kubenest/tasks/anp.go index 50d4bb73e..653b9a4cd 100644 --- a/pkg/kubenest/tasks/anp.go +++ b/pkg/kubenest/tasks/anp.go @@ -177,7 +177,7 @@ func installAnpServer(client clientset.Interface, name, namespace string, portMa AdmissionPlugins bool IPV6First bool }{ - DeploymentName: fmt.Sprintf("%s-%s", name, "apiserver"), + DeploymentName: fmt.Sprintf("%s-%s", name, "apiserver-anp"), Namespace: namespace, ImageRepository: imageRepository, Version: imageVersion, @@ -188,7 +188,6 @@ func installAnpServer(client clientset.Interface, name, namespace string, portMa EtcdCertsSecret: fmt.Sprintf("%s-%s", name, "etcd-cert"), Replicas: kubeNestConfiguration.KubeInKubeConfig.ApiServerReplicas, EtcdListenClientPort: constants.ApiServerEtcdListenClientPort, - ClusterPort: portMap[constants.ApiServerPortKey], AgentPort: portMap[constants.ApiServerNetworkProxyAgentPortKey], ServerPort: portMap[constants.ApiServerNetworkProxyServerPortKey], HealthPort: portMap[constants.ApiServerNetworkProxyHealthPortKey],