diff --git a/pkg/kubenest/controlplane/apiserver.go b/pkg/kubenest/controlplane/apiserver.go index b998cec57..72a75aadd 100644 --- a/pkg/kubenest/controlplane/apiserver.go +++ b/pkg/kubenest/controlplane/apiserver.go @@ -34,7 +34,7 @@ func DeleteVirtualClusterAPIServer(client clientset.Interface, name, namespace s } err := manager.ReleaseHostPort(name) if err != nil { - klog.Errorf("Error releasing host port for cluster %s: %v", name, err) + klog.Warningf("Error releasing host port for cluster %s: %v", name, err) } return nil } diff --git a/pkg/kubenest/controlplane/component.go b/pkg/kubenest/controlplane/component.go index 85269fc02..bba8acef3 100644 --- a/pkg/kubenest/controlplane/component.go +++ b/pkg/kubenest/controlplane/component.go @@ -123,7 +123,7 @@ func getKubeControllerManagerManifest(name, namespace string) (*appsv1.Deploymen ImageRepository: imageRepository, Version: imageVersion, VirtualClusterCertsSecret: fmt.Sprintf("%s-%s", name, "cert"), - KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config"), + KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config-clusterip"), ServiceSubnet: constants.ApiServerServiceSubnet, Replicas: constants.KubeControllerReplicas, }) @@ -170,7 +170,7 @@ func getVirtualClusterSchedulerManifest(name, namespace string) (*appsv1.Deploym SystemNamespace: constants.SystemNs, ImageRepository: imageRepository, Version: imageVersion, - KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config"), + KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config-clusterip"), Replicas: constants.VirtualClusterSchedulerReplicas, }) if err != nil { diff --git a/pkg/kubenest/controlplane/service.go b/pkg/kubenest/controlplane/service.go index 0b95d3a21..ca95e3655 100644 --- a/pkg/kubenest/controlplane/service.go +++ b/pkg/kubenest/controlplane/service.go @@ -50,7 +50,7 @@ func DeleteVirtualClusterService(client clientset.Interface, name, namespace str err := manager.ReleaseHostPort(name) if err != nil { - klog.Errorf("Error releasing host port for cluster %s: %v", name, err) + klog.Warningf("Error releasing host port for cluster %s: %v", name, err) } klog.V(2).Infof("Successfully uninstalled service for virtualcluster %s", name) diff --git a/pkg/kubenest/tasks/upload.go b/pkg/kubenest/tasks/upload.go index af86052be..5282f6e5f 100644 --- a/pkg/kubenest/tasks/upload.go +++ b/pkg/kubenest/tasks/upload.go @@ -24,6 +24,11 @@ var ( VirtualClusterControllerLabel = labels.Set{constants.VirtualClusterLabelKeyName: constants.VirtualClusterController} ) +type PortInfo struct { + NodePort int32 + ClusterIPPort int32 +} + func NewUploadCertsTask() workflow.Task { return workflow.Task{ Name: "Upload-Certs", @@ -165,19 +170,32 @@ func runUploadAdminKubeconfig(r workflow.RunData) error { return errors.New("UploadAdminKubeconfig task invoked with an invalid data struct") } - var endpoint string + var endpointControlplaneIp, endpointClusterIP string service, err := data.RemoteClient().CoreV1().Services(data.GetNamespace()).Get(context.TODO(), fmt.Sprintf("%s-%s", data.GetName(), "apiserver"), metav1.GetOptions{}) if err != nil { return err } - nodePort := getNodePortFromAPIServerService(service) - endpoint = fmt.Sprintf("https://%s:%d", data.ControlplaneAddress(), nodePort) - kubeconfig, err := buildKubeConfigFromSpec(data, endpoint) + portInfo := getPortInfoFromAPIServerService(service) + // controlplane address + nodePort + endpointControlplaneIp = fmt.Sprintf("https://%s:%d", data.ControlplaneAddress(), portInfo.NodePort) + kubeconfigControlplaneIp, err := buildKubeConfigFromSpec(data, endpointControlplaneIp) + if err != nil { + return err + } + + //clusterIP address + clusterIPPort + endpointClusterIP = fmt.Sprintf("https://%s:%d", service.Spec.ClusterIP, portInfo.ClusterIPPort) + kubeconfigClusterIP, err := buildKubeConfigFromSpec(data, endpointClusterIP) + if err != nil { + return err + } + + configBytesControlplaneIp, err := clientcmd.Write(*kubeconfigControlplaneIp) if err != nil { return err } - configBytes, err := clientcmd.Write(*kubeconfig) + configBytesClusterIP, err := clientcmd.Write(*kubeconfigClusterIP) if err != nil { return err } @@ -188,28 +206,42 @@ func runUploadAdminKubeconfig(r workflow.RunData) error { Name: fmt.Sprintf("%s-%s", data.GetName(), "admin-config"), Labels: VirtualClusterControllerLabel, }, - Data: map[string][]byte{"kubeconfig": configBytes}, + Data: map[string][]byte{"kubeconfig": configBytesControlplaneIp}, }) if err != nil { return fmt.Errorf("failed to create secret of kubeconfig, err: %w", err) } - klog.V(2).InfoS("[UploadAdminKubeconfig] Successfully created secret of virtual cluster apiserver kubeconfig", "virtual cluster", klog.KObj(data)) + err = createOrUpdateSecret(data.RemoteClient(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: data.GetNamespace(), + Name: fmt.Sprintf("%s-%s", data.GetName(), "admin-config-clusterip"), + Labels: VirtualClusterControllerLabel, + }, + Data: map[string][]byte{"kubeconfig": configBytesClusterIP}, + }) + if err != nil { + return fmt.Errorf("failed to create secret of kubeconfig-clusterip, err: %w", err) + } + + klog.V(2).InfoS("[UploadAdminKubeconfig] Successfully created secrets of virtual cluster apiserver kubeconfig", "virtual cluster", klog.KObj(data)) return nil } -func getNodePortFromAPIServerService(service *corev1.Service) int32 { - var nodePort int32 +func getPortInfoFromAPIServerService(service *corev1.Service) PortInfo { + var portInfo PortInfo + //var nodePort int32 if service.Spec.Type == corev1.ServiceTypeNodePort { for _, port := range service.Spec.Ports { if port.Name != constants.APIServerSVCPortName { continue } - nodePort = port.NodePort + portInfo.NodePort = port.NodePort + portInfo.ClusterIPPort = port.Port } } - return nodePort + return portInfo } func buildKubeConfigFromSpec(data InitData, serverURL string) (*clientcmdapi.Config, error) { @@ -272,6 +304,7 @@ func deleteSecrets(r workflow.RunData) error { fmt.Sprintf("%s-%s", data.GetName(), "cert"), fmt.Sprintf("%s-%s", data.GetName(), "etcd-cert"), fmt.Sprintf("%s-%s", data.GetName(), "admin-config"), + fmt.Sprintf("%s-%s", data.GetName(), "admin-config-clusterip"), } for _, secret := range secrets { err := data.RemoteClient().CoreV1().Secrets(data.GetNamespace()).Delete(context.TODO(), secret, metav1.DeleteOptions{})