fix: Modify the ip in the kubeconfig used by the host cluster to clus…

…ter ip

Signed-off-by: qiuwei <[email protected]>

pkg/kubenest/controlplane/apiserver.go

@@ -35,7 +35,7 @@ func DeleteVirtualClusterAPIServer(client clientset.Interface, name, namespace s
 	}
 	err := manager.ReleaseHostPort(name)
 	if err != nil {
-		klog.Errorf("Error releasing host port for cluster %s: %v", name, err)
+		klog.Warningf("Error releasing host port for cluster %s: %v", name, err)
 	}
 	return nil
 }

pkg/kubenest/controlplane/component.go

@@ -124,7 +124,7 @@ func getKubeControllerManagerManifest(name, namespace string) (*appsv1.Deploymen
 		ImageRepository:           imageRepository,
 		Version:                   imageVersion,
 		VirtualClusterCertsSecret: fmt.Sprintf("%s-%s", name, "cert"),
-		KubeconfigSecret:          fmt.Sprintf("%s-%s", name, "admin-config"),
+		KubeconfigSecret:          fmt.Sprintf("%s-%s", name, "admin-config-clusterip"),
 		ServiceSubnet:             constants.ApiServerServiceSubnet,
 		Replicas:                  constants.KubeControllerReplicas,
 	})
@@ -171,7 +171,7 @@ func getVirtualClusterSchedulerManifest(name, namespace string) (*appsv1.Deploym
 		SystemNamespace:  constants.SystemNs,
 		ImageRepository:  imageRepository,
 		Version:          imageVersion,
-		KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config"),
+		KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config-clusterip"),
 		Replicas:         constants.VirtualClusterSchedulerReplicas,
 	})
 	if err != nil {

pkg/kubenest/controlplane/service.go

@@ -51,7 +51,7 @@ func DeleteVirtualClusterService(client clientset.Interface, name, namespace str
 
 	err := manager.ReleaseHostPort(name)
 	if err != nil {
-		klog.Errorf("Error releasing host port for cluster %s: %v", name, err)
+		klog.Warningf("Error releasing host port for cluster %s: %v", name, err)
 	}
 
 	klog.V(2).Infof("Successfully uninstalled service for virtualcluster %s", name)

pkg/kubenest/tasks/upload.go

@@ -24,6 +24,11 @@ var (
 	VirtualClusterControllerLabel = labels.Set{constants.VirtualClusterLabelKeyName: constants.VirtualClusterController}
 )
 
+type PortInfo struct {
+	NodePort      int32
+	ClusterIPPort int32
+}
+
 func NewUploadCertsTask() workflow.Task {
 	return workflow.Task{
 		Name:        "Upload-Certs",
@@ -165,51 +170,82 @@ func runUploadAdminKubeconfig(r workflow.RunData) error {
 		return errors.New("UploadAdminKubeconfig task invoked with an invalid data struct")
 	}
 
-	var endpoint string
+	var errs error
+	var endpointControlplaneIp, endpointClusterIP string
 	service, err := data.RemoteClient().CoreV1().Services(data.GetNamespace()).Get(context.TODO(), fmt.Sprintf("%s-%s", data.GetName(), "apiserver"), metav1.GetOptions{})
 	if err != nil {
 		return err
 	}
-	nodePort := getNodePortFromAPIServerService(service)
-	endpoint = fmt.Sprintf("https://%s:%d", data.ControlplaneAddress(), nodePort)
-	kubeconfig, err := buildKubeConfigFromSpec(data, endpoint)
+	portInfo := getPortInfoFromAPIServerService(service)
+	// controlplane address + nodePort
+	endpointControlplaneIp = fmt.Sprintf("https://%s:%d", data.ControlplaneAddress(), portInfo.NodePort)
+	kubeconfigControlplaneIp, err := buildKubeConfigFromSpec(data, endpointControlplaneIp)
+	if err != nil {
+		return err
+	}
+
+	//clusterIP address + clusterIPPort
+	endpointClusterIP = fmt.Sprintf("https://%s:%d", service.Spec.ClusterIP, portInfo.ClusterIPPort)
+	kubeconfigClusterIP, err := buildKubeConfigFromSpec(data, endpointClusterIP)
+	if err != nil {
+		return err
+	}
+
+	configBytesControlplaneIp, err := clientcmd.Write(*kubeconfigControlplaneIp)
 	if err != nil {
 		return err
 	}
 
-	configBytes, err := clientcmd.Write(*kubeconfig)
+	configBytesClusterIP, err := clientcmd.Write(*kubeconfigClusterIP)
 	if err != nil {
 		return err
 	}
 
-	err = createOrUpdateSecret(data.RemoteClient(), &corev1.Secret{
+	errControlplaneIp := createOrUpdateSecret(data.RemoteClient(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: data.GetNamespace(),
 			Name:      fmt.Sprintf("%s-%s", data.GetName(), "admin-config"),
 			Labels:    VirtualClusterControllerLabel,
 		},
-		Data: map[string][]byte{"kubeconfig": configBytes},
+		Data: map[string][]byte{"kubeconfig": configBytesControlplaneIp},
 	})
-	if err != nil {
-		return fmt.Errorf("failed to create secret of kubeconfig, err: %w", err)
+	if errControlplaneIp != nil {
+		errs = errors.Wrap(errControlplaneIp, fmt.Sprint(errs))
 	}
 
-	klog.V(2).InfoS("[UploadAdminKubeconfig] Successfully created secret of virtual cluster apiserver kubeconfig", "virtual cluster", klog.KObj(data))
+	errClusterIP := createOrUpdateSecret(data.RemoteClient(), &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: data.GetNamespace(),
+			Name:      fmt.Sprintf("%s-%s", data.GetName(), "admin-config-clusterip"),
+			Labels:    VirtualClusterControllerLabel,
+		},
+		Data: map[string][]byte{"kubeconfig": configBytesClusterIP},
+	})
+	if errClusterIP != nil {
+		errs = errors.Wrap(errClusterIP, fmt.Sprint(errs))
+	}
+
+	if errs != nil {
+		return errs
+	}
+	klog.V(2).InfoS("[UploadAdminKubeconfig] Successfully created secrets of virtual cluster apiserver kubeconfig", "virtual cluster", klog.KObj(data))
 	return nil
 }
 
-func getNodePortFromAPIServerService(service *corev1.Service) int32 {
-	var nodePort int32
+func getPortInfoFromAPIServerService(service *corev1.Service) PortInfo {
+	var portInfo PortInfo
+	//var nodePort int32
 	if service.Spec.Type == corev1.ServiceTypeNodePort {
 		for _, port := range service.Spec.Ports {
 			if port.Name != constants.APIServerSVCPortName {
 				continue
 			}
-			nodePort = port.NodePort
+			portInfo.NodePort = port.NodePort
+			portInfo.ClusterIPPort = port.Port
 		}
 	}
 
-	return nodePort
+	return portInfo
 }
 
 func buildKubeConfigFromSpec(data InitData, serverURL string) (*clientcmdapi.Config, error) {
@@ -272,6 +308,7 @@ func deleteSecrets(r workflow.RunData) error {
 		fmt.Sprintf("%s-%s", data.GetName(), "cert"),
 		fmt.Sprintf("%s-%s", data.GetName(), "etcd-cert"),
 		fmt.Sprintf("%s-%s", data.GetName(), "admin-config"),
+		fmt.Sprintf("%s-%s", data.GetName(), "admin-config-clusterip"),
 	}
 	for _, secret := range secrets {
 		err := data.RemoteClient().CoreV1().Secrets(data.GetNamespace()).Delete(context.TODO(), secret, metav1.DeleteOptions{})