diff --git a/pkg/clustertree/cluster-manager/controllers/common_controller.go b/pkg/clustertree/cluster-manager/controllers/common_controller.go index b86def352..3eba7d8af 100644 --- a/pkg/clustertree/cluster-manager/controllers/common_controller.go +++ b/pkg/clustertree/cluster-manager/controllers/common_controller.go @@ -118,14 +118,21 @@ func (r *SyncResourcesReconciler) SyncResource(ctx context.Context, request reco klog.V(4).Infof("Started sync resource processing, ns: %s, name: %s", request.Namespace, request.Name) deleteSecretInClient := false + resourceNamespace := request.Namespace + masterResourceName := request.Name + memberResourceName := masterResourceName + // The name of the host cluster kube-root-ca.crt in the leaf group is master-root-ca.crt + if r.GroupVersionResource == utils.GVR_CONFIGMAP && masterResourceName == utils.RooTCAConfigMapName { + memberResourceName = utils.MasterRooTCAName + } - obj, err := r.DynamicRootClient.Resource(r.GroupVersionResource).Namespace(request.Namespace).Get(ctx, request.Name, metav1.GetOptions{}) + obj, err := r.DynamicRootClient.Resource(r.GroupVersionResource).Namespace(resourceNamespace).Get(ctx, masterResourceName, metav1.GetOptions{}) if err != nil { if !errors.IsNotFound(err) { return err } // get obj in leaf cluster - _, err := lr.DynamicClient.Resource(r.GroupVersionResource).Namespace(request.Namespace).Get(ctx, request.Name, metav1.GetOptions{}) + _, err := lr.DynamicClient.Resource(r.GroupVersionResource).Namespace(resourceNamespace).Get(ctx, memberResourceName, metav1.GetOptions{}) if err != nil { if !errors.IsNotFound(err) { klog.Errorf("Get %s from leaef cluster failed, error: %v", obj.GetKind(), err) @@ -140,17 +147,17 @@ func (r *SyncResourcesReconciler) SyncResource(ctx context.Context, request reco if deleteSecretInClient || obj.GetDeletionTimestamp() != nil { // delete OBJ in leaf cluster - if err = lr.DynamicClient.Resource(r.GroupVersionResource).Namespace(request.Namespace).Delete(ctx, request.Name, metav1.DeleteOptions{}); err != nil { + if err = lr.DynamicClient.Resource(r.GroupVersionResource).Namespace(resourceNamespace).Delete(ctx, memberResourceName, metav1.DeleteOptions{}); err != nil { if errors.IsNotFound(err) { return nil } return err } - klog.V(4).Infof("%s %q deleted", r.GroupVersionResource.Resource, request.Name) + klog.V(4).Infof("%s %q deleted in member cluster", r.GroupVersionResource.Resource, memberResourceName) return nil } - old, err := lr.DynamicClient.Resource(r.GroupVersionResource).Namespace(request.Namespace).Get(ctx, request.Name, metav1.GetOptions{}) + old, err := lr.DynamicClient.Resource(r.GroupVersionResource).Namespace(resourceNamespace).Get(ctx, memberResourceName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { @@ -177,7 +184,7 @@ func (r *SyncResourcesReconciler) SyncResource(ctx context.Context, request reco if !utils.IsObjectUnstructuredGlobal(old.GetAnnotations()) { return nil } - _, err = lr.DynamicClient.Resource(r.GroupVersionResource).Namespace(request.Namespace).Update(ctx, latest, metav1.UpdateOptions{}) + _, err = lr.DynamicClient.Resource(r.GroupVersionResource).Namespace(resourceNamespace).Update(ctx, latest, metav1.UpdateOptions{}) if err != nil { klog.Errorf("update %s from client cluster failed, error: %v", latest.GetKind(), err) return err diff --git a/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go b/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go index 9fea35173..30cea2269 100644 --- a/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go +++ b/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go @@ -303,22 +303,24 @@ func (r *RootPodReconciler) createStorageInLeafCluster(ctx context.Context, lr * } // create resource in leaf cluster - _, err = lr.DynamicClient.Resource(gvr).Namespace(ns).Get(ctx, rname, metav1.GetOptions{}) + unstructuredObj := rootobj + + podutils.FitUnstructuredObjMeta(unstructuredObj) + if err := storageHandler.BeforeGetInLeaf(ctx, r, lr, unstructuredObj, rootpod, cn); err != nil { + return err + } + + _, err = lr.DynamicClient.Resource(gvr).Namespace(ns).Get(ctx, unstructuredObj.GetName(), metav1.GetOptions{}) if err == nil { // already existed, so skip continue } if errors.IsNotFound(err) { - unstructuredObj := rootobj - - podutils.FitUnstructuredObjMeta(unstructuredObj) - + podutils.SetUnstructuredObjGlobal(unstructuredObj) if err := storageHandler.BeforeCreateInLeaf(ctx, r, lr, unstructuredObj, rootpod, cn); err != nil { return err } - podutils.SetUnstructuredObjGlobal(unstructuredObj) - _, err = lr.DynamicClient.Resource(gvr).Namespace(ns).Create(ctx, unstructuredObj, metav1.CreateOptions{}) if err != nil { if errors.IsAlreadyExists(err) { @@ -335,37 +337,9 @@ func (r *RootPodReconciler) createStorageInLeafCluster(ctx context.Context, lr * return nil } -func (r *RootPodReconciler) createSAInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, sa string, ns string) (*corev1.ServiceAccount, error) { - saKey := types.NamespacedName{ - Namespace: ns, - Name: sa, - } - - clientSA := &corev1.ServiceAccount{} - err := lr.Client.Get(ctx, saKey, clientSA) - if err != nil && !errors.IsNotFound(err) { - return nil, fmt.Errorf("could not check sa %s in member cluster: %v", sa, err) - } - - if err == nil { - return clientSA, nil - } - - newSA := &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: sa, - Namespace: ns, - }, - } - err = lr.Client.Create(ctx, newSA) - if err != nil && !errors.IsAlreadyExists(err) { - return nil, fmt.Errorf("could not create sa %s in member cluster: %v", sa, err) - } - - return newSA, nil -} - -func (r *RootPodReconciler) createSATokenInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, saName string, ns string) (*corev1.Secret, error) { +func (r *RootPodReconciler) createSATokenInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, saName string, pod *corev1.Pod) (string, error) { + // get the token-secret of sa + ns := pod.Namespace satokenKey := types.NamespacedName{ Namespace: ns, Name: saName, @@ -373,95 +347,185 @@ func (r *RootPodReconciler) createSATokenInLeafCluster(ctx context.Context, lr * sa := &corev1.ServiceAccount{} err := r.RootClient.Get(ctx, satokenKey, sa) if err != nil { - return nil, fmt.Errorf("could not find sa %s in master cluster: %v", saName, err) + return "", fmt.Errorf("could not find sa %s in master cluster: %v", saName, err) } - var secretName string + var rootSecretName string if len(sa.Secrets) > 0 { - secretName = sa.Secrets[0].Name + rootSecretName = sa.Secrets[0].Name + } + + if rootSecretName == "" { + // k8s version >=1.24, sa does not create token-secret by default, + // so we will create and mount it to the subset group + tokenSecret, err := r.createTokenSecretInRootCluster(ctx, sa) + if err != nil { + return "", err + } + + rootSecretName = tokenSecret.Name } - csName := fmt.Sprintf("master-%s-token", sa.Name) csKey := types.NamespacedName{ Namespace: ns, - Name: csName, + Name: rootSecretName, } clientSecret := &corev1.Secret{} err = lr.Client.Get(ctx, csKey, clientSecret) if err != nil && !errors.IsNotFound(err) { - return nil, fmt.Errorf("could not check secret %s in member cluster: %v", secretName, err) + return "", fmt.Errorf("could not check secret %s in member cluster: %v", csKey.Name, err) } if err == nil { - return clientSecret, nil + return clientSecret.Name, nil } - secretKey := types.NamespacedName{ - Namespace: ns, - Name: secretName, - } + // this secret needs to be created in member cluster + ch := make(chan string, 1) + clusterNodeInfo := r.GlobalLeafManager.GetClusterNode(pod.Spec.NodeName) + go func() { + if err = wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) { + if err := r.createStorageInLeafCluster(ctx, lr, utils.GVR_SECRET, []string{rootSecretName}, pod, clusterNodeInfo); err == nil { + return true, nil + } else { + return false, err + } + }); err != nil { + ch <- fmt.Sprintf("could not create secret token %s in leaf cluster: %v", rootSecretName, err) + } + ch <- "" + }() - masterSecret := &corev1.Secret{} - err = r.RootClient.Get(ctx, secretKey, masterSecret) - if err != nil { - return nil, fmt.Errorf("could not find secret %s in master cluster: %v", secretName, err) + t := <-ch + errString := "" + t + if len(errString) > 0 { + return "", fmt.Errorf("%s", errString) } - nData := map[string][]byte{} - nData["token"] = masterSecret.Data["token"] + return rootSecretName, nil +} - newSE := &corev1.Secret{ +func (r *RootPodReconciler) createTokenSecretInRootCluster(ctx context.Context, sa *corev1.ServiceAccount) (*corev1.Secret, error) { + tokenSecretName := fmt.Sprintf("kosmos-%s-token", sa.Name) + tokenSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: csName, - Namespace: ns, + Name: tokenSecretName, + Namespace: sa.Namespace, + Annotations: map[string]string{ + corev1.ServiceAccountNameKey: sa.Name, + corev1.ServiceAccountUIDKey: string(sa.UID), + }, }, - Data: nData, + Type: corev1.SecretTypeServiceAccountToken, } - err = lr.Client.Create(ctx, newSE) + err := r.RootClient.Create(ctx, tokenSecret) - if err != nil && !errors.IsAlreadyExists(err) { - return nil, fmt.Errorf("could not create sa %s in member cluster: %v", sa, err) + if err != nil { + return nil, fmt.Errorf("could not create token-secret %s in host cluster: %v", tokenSecretName, err) } - return newSE, nil + + // Attach token-secret to sa + patchSa := sa.DeepCopy() + patchSa.Secrets = []corev1.ObjectReference{ + { + Name: tokenSecretName, + }, + } + err = r.RootClient.Update(ctx, patchSa) + if err != nil { + return nil, fmt.Errorf("could not update sa %s in host cluster: %v", patchSa.Name, err) + } + + return tokenSecret, nil } -func (r *RootPodReconciler) createCAInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, ns string) (*corev1.ConfigMap, error) { - masterCAConfigmapKey := types.NamespacedName{ +// createConfigMapInLeafCluster create cm in leaf cluster +func (r *RootPodReconciler) createConfigMapInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, configMapName string, pod *corev1.Pod) (string, error) { + ns := pod.Namespace + memberConfigmapKeyName := configMapName + + // The name of the host cluster kube-root-ca.crt in the leaf group is master-root-ca.crt + if strings.HasPrefix(configMapName, utils.RooTCAConfigMapName) { + memberConfigmapKeyName = utils.MasterRooTCAName + } + + configmapKey := types.NamespacedName{ Namespace: ns, - Name: utils.MasterRooTCAName, + Name: memberConfigmapKeyName, } - masterCA := &corev1.ConfigMap{} + memberConfigMap := &corev1.ConfigMap{} - err := lr.Client.Get(ctx, masterCAConfigmapKey, masterCA) + err := lr.Client.Get(ctx, configmapKey, memberConfigMap) if err != nil && !errors.IsNotFound(err) { - return nil, fmt.Errorf("could not check configmap %s in member cluster: %v", utils.MasterRooTCAName, err) + return "", fmt.Errorf("could not check configmap %s in member cluster: %v", configmapKey.Name, err) } if err == nil { - return masterCA, nil + return memberConfigMap.Name, nil } - ca := &corev1.ConfigMap{} + ch := make(chan string, 1) + clusterNodeInfo := r.GlobalLeafManager.GetClusterNode(pod.Spec.NodeName) + go func() { + if err = wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) { + if err = r.createStorageInLeafCluster(ctx, lr, utils.GVR_CONFIGMAP, []string{configMapName}, pod, clusterNodeInfo); err == nil { + return true, nil + } else { + return false, err + } + }); err != nil { + ch <- fmt.Sprintf("could not create configmap %s in member cluster: %v", configMapName, err) + } + ch <- "" + }() - rootCAConfigmapKey := types.NamespacedName{ + t := <-ch + errString := "" + t + if len(errString) > 0 { + return "", fmt.Errorf("%s", errString) + } + + return memberConfigmapKeyName, nil +} + +func (r *RootPodReconciler) createSecretInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, secretName string, pod *corev1.Pod) (string, error) { + ns := pod.Namespace + secretKey := types.NamespacedName{ Namespace: ns, - Name: utils.RooTCAConfigMapName, + Name: secretName, } - err = r.Client.Get(ctx, rootCAConfigmapKey, ca) - if err != nil { - return nil, fmt.Errorf("could not find configmap %s in master cluster: %v", ca, err) + memberSecret := &corev1.Secret{} + + err := lr.Client.Get(ctx, secretKey, memberSecret) + if err != nil && !errors.IsNotFound(err) { + return "", fmt.Errorf("could not check secret %s in member cluster: %v", secretKey.Name, err) + } + if err == nil { + return memberSecret.Name, nil } - newCA := ca.DeepCopy() - newCA.Name = utils.MasterRooTCAName - podutils.FitObjectMeta(&newCA.ObjectMeta) + ch := make(chan string, 1) + clusterNodeInfo := r.GlobalLeafManager.GetClusterNode(pod.Spec.NodeName) + go func() { + if err = wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) { + if err = r.createStorageInLeafCluster(ctx, lr, utils.GVR_SECRET, []string{secretName}, pod, clusterNodeInfo); err == nil { + return true, nil + } else { + return false, err + } + }); err != nil { + ch <- fmt.Sprintf("could not create secret %s in member cluster: %v", secretName, err) + } + ch <- "" + }() - err = lr.Client.Create(ctx, newCA) - if err != nil && !errors.IsAlreadyExists(err) { - return nil, fmt.Errorf("could not create configmap %s in member cluster: %v", newCA.Name, err) + t := <-ch + errString := "" + t + if len(errString) > 0 { + return "", fmt.Errorf("%s", errString) } - return newCA, nil + return secretName, nil } // changeToMasterCoreDNS point the dns of the pod to the master cluster, so that the pod can access any service. @@ -495,62 +559,76 @@ func (r *RootPodReconciler) changeToMasterCoreDNS(ctx context.Context, pod *core } } -func (r *RootPodReconciler) convertAuth(ctx context.Context, lr *leafUtils.LeafResource, pod *corev1.Pod) { - if pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken { - falseValue := false - pod.Spec.AutomountServiceAccountToken = &falseValue - - sa := pod.Spec.ServiceAccountName - _, err := r.createSAInLeafCluster(ctx, lr, sa, pod.Namespace) - if err != nil { - klog.Errorf("[convertAuth] create sa failed, ns: %s, pod: %s", pod.Namespace, pod.Name) - return - } +// projectedHandler Process the project volume, creating and mounting secret, configmap, DownwardAPI, +// and ServiceAccountToken from the project volume in the member cluster to the pod of the host cluster +func (r *RootPodReconciler) projectedHandler(ctx context.Context, lr *leafUtils.LeafResource, pod *corev1.Pod) { + if pod.Spec.Volumes == nil { + return + } - se, err := r.createSATokenInLeafCluster(ctx, lr, sa, pod.Namespace) - if err != nil { - klog.Errorf("[convertAuth] create sa secret failed, ns: %s, pod: %s", pod.Namespace, pod.Name) - return - } + for _, volume := range pod.Spec.Volumes { + if volume.Projected != nil { + falseValue := false + pod.Spec.AutomountServiceAccountToken = &falseValue - rootCA, err := r.createCAInLeafCluster(ctx, lr, pod.Namespace) - if err != nil { - klog.Errorf("[convertAuth] create sa secret failed, ns: %s, pod: %s", pod.Namespace, pod.Name) - return - } + saName := pod.Spec.ServiceAccountName + var sources []corev1.VolumeProjection - volumes := pod.Spec.Volumes - for _, v := range volumes { - if strings.HasPrefix(v.Name, utils.SATokenPrefix) { - sources := []corev1.VolumeProjection{} - for _, src := range v.Projected.Sources { - if src.ServiceAccountToken != nil { - continue - } - if src.ConfigMap != nil && src.ConfigMap.Name == utils.RooTCAConfigMapName { - src.ConfigMap.Name = rootCA.Name + for _, projectedVolumeSource := range volume.Projected.Sources { + // Process all resources for the rootpod + if projectedVolumeSource.ServiceAccountToken != nil { + tokenSecretName, err := r.createSATokenInLeafCluster(ctx, lr, saName, pod) + if err != nil { + klog.Errorf("[convertAuth] create sa secret failed, ns: %s, pod: %s, err: %s", pod.Namespace, pod.Name, err) + return } - sources = append(sources, src) - } - - secretProjection := corev1.VolumeProjection{ - Secret: &corev1.SecretProjection{ - Items: []corev1.KeyToPath{ - { - Key: "token", - Path: "token", + secretProjection := corev1.VolumeProjection{ + Secret: &corev1.SecretProjection{ + Items: []corev1.KeyToPath{ + { + Key: "token", + Path: projectedVolumeSource.ServiceAccountToken.Path, + }, }, }, - }, + } + secretProjection.Secret.Name = tokenSecretName + sources = append(sources, secretProjection) + } + if projectedVolumeSource.ConfigMap != nil { + cmName, err := r.createConfigMapInLeafCluster(ctx, lr, projectedVolumeSource.ConfigMap.Name, pod) + if err != nil { + klog.Errorf("[convertAuth] create configmap failed, ns: %s, cm: %s, err: %s", pod.Namespace, cmName, err) + return + } + cmDeepCopy := projectedVolumeSource.DeepCopy() + cmDeepCopy.ConfigMap.Name = cmName + sources = append(sources, *cmDeepCopy) + } + if projectedVolumeSource.Secret != nil { + Secret := projectedVolumeSource.Secret + seName, err := r.createSecretInLeafCluster(ctx, lr, Secret.Name, pod) + if err != nil { + klog.Errorf("[convertAuth] create secret failed, ns: %s, cm: %s, err: %s", pod.Namespace, seName, err) + return + } + secretDeepCopy := projectedVolumeSource.DeepCopy() + secretDeepCopy.Secret.Name = seName + sources = append(sources, *secretDeepCopy) + } + if projectedVolumeSource.DownwardAPI != nil { + DownwardAPIProjection := corev1.VolumeProjection{ + DownwardAPI: projectedVolumeSource.DownwardAPI, + } + sources = append(sources, DownwardAPIProjection) } - secretProjection.Secret.Name = se.Name - sources = append(sources, secretProjection) - v.Projected.Sources = sources } + volume.Projected.Sources = sources } } } +// createServiceAccountInLeafCluster Create an sa corresponding to token-secret in member cluster func (r *RootPodReconciler) createServiceAccountInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, secret *corev1.Secret) error { if !lr.EnableServiceAccount { return nil @@ -561,46 +639,54 @@ func (r *RootPodReconciler) createServiceAccountInLeafCluster(ctx context.Contex klog.V(4).Infof("secret service-account info: [%v]", secret.Annotations) accountName := secret.Annotations[corev1.ServiceAccountNameKey] if accountName == "" { - err := fmt.Errorf("get secret of serviceAccount not exits: [%s] [%v]", + err := fmt.Errorf("get serviceAccount of secret not exits: [%s] [%v]", secret.Name, secret.Annotations) return err } - ns := secret.Namespace sa := &corev1.ServiceAccount{} - saKey := types.NamespacedName{ - Namespace: ns, - Name: accountName, - } - - err := lr.Client.Get(ctx, saKey, sa) - if err != nil || sa == nil { - klog.V(4).Infof("get serviceAccount [%v] err: [%v]]", sa, err) - sa = &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: accountName, - Namespace: ns, - }, + if accountName != utils.DefaultServiceAccountName { + ns := secret.Namespace + saKey := types.NamespacedName{ + Namespace: ns, + Name: accountName, } - err := lr.Client.Create(ctx, sa) - klog.Errorf("create serviceAccount [%v] err: [%v]", sa, err) - if err != nil { - if errors.IsAlreadyExists(err) { - return nil + + err := lr.Client.Get(ctx, saKey, sa) + if err != nil || sa == nil { + klog.V(4).Infof("get serviceAccount [%v] err: [%v]]", sa, err) + sa = &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: accountName, + Namespace: ns, + }, } - return err + err := lr.Client.Create(ctx, sa) + klog.Errorf("create serviceAccount [%v] err: [%v]", sa, err) + if err != nil { + if errors.IsAlreadyExists(err) { + return nil + } + return err + } + } else { + klog.V(4).Infof("get secret serviceAccount info: [%s] [%v] [%v] [%v]", + sa.Name, sa.CreationTimestamp, sa.Annotations, sa.UID) } + + secret.UID = sa.UID + secret.Annotations[corev1.ServiceAccountNameKey] = accountName + secret.Annotations[corev1.ServiceAccountUIDKey] = string(sa.UID) + + secret.ObjectMeta.Namespace = ns } else { - klog.V(4).Infof("get secret serviceAccount info: [%s] [%v] [%v] [%v]", - sa.Name, sa.CreationTimestamp, sa.Annotations, sa.UID) + // accountName == default + // Type set Opaque and add annotation of kosmos.io/service-account.name + secret.Annotations[utils.DefaultServiceAccountToken] = utils.DefaultServiceAccountName + secret.Type = corev1.SecretTypeOpaque } - secret.UID = sa.UID - secret.Annotations[corev1.ServiceAccountNameKey] = accountName - secret.Annotations[corev1.ServiceAccountUIDKey] = string(sa.UID) - - secret.ObjectMeta.Namespace = ns - err = lr.Client.Create(ctx, secret) + err := lr.Client.Create(ctx, secret) if err != nil { if errors.IsAlreadyExists(err) { @@ -609,15 +695,19 @@ func (r *RootPodReconciler) createServiceAccountInLeafCluster(ctx context.Contex klog.Errorf("Failed to create secret %v err: %v", secret.Name, err) } - sa.Secrets = []corev1.ObjectReference{{Name: secret.Name}} + // the secret-token cannot be mounted to the default-sa of the leaf cluster + if accountName != utils.DefaultServiceAccountName { + sa.Secrets = []corev1.ObjectReference{{Name: secret.Name}} - err = lr.Client.Update(ctx, sa) - if err != nil { - klog.V(4).Infof( - "update serviceAccount [%v] err: [%v]]", - sa, err) - return err + err = lr.Client.Update(ctx, sa) + if err != nil { + klog.V(4).Infof( + "update serviceAccount [%v] err: [%v]]", + sa, err) + return err + } } + return nil } @@ -751,7 +841,7 @@ func (r *RootPodReconciler) CreatePodInLeafCluster(ctx context.Context, lr *leaf klog.V(4).Infof("Creating Volumes successed %+v", basicPod) } - r.convertAuth(ctx, lr, basicPod) + r.projectedHandler(ctx, lr, basicPod) if !r.Options.MultiClusterService { r.changeToMasterCoreDNS(ctx, basicPod, r.Options) @@ -792,7 +882,7 @@ func (r *RootPodReconciler) UpdatePodInLeafCluster(ctx context.Context, lr *leaf return nil } - r.convertAuth(ctx, lr, podCopy) + r.projectedHandler(ctx, lr, podCopy) if !r.Options.MultiClusterService { r.changeToMasterCoreDNS(ctx, podCopy, r.Options) diff --git a/pkg/clustertree/cluster-manager/controllers/pod/storage_handler.go b/pkg/clustertree/cluster-manager/controllers/pod/storage_handler.go index 9e4c06565..5a61aa19e 100644 --- a/pkg/clustertree/cluster-manager/controllers/pod/storage_handler.go +++ b/pkg/clustertree/cluster-manager/controllers/pod/storage_handler.go @@ -16,6 +16,7 @@ import ( type StorageHandler interface { BeforeCreateInLeaf(context.Context, *RootPodReconciler, *leafUtils.LeafResource, *unstructured.Unstructured, *corev1.Pod, *leafUtils.ClusterNode) error + BeforeGetInLeaf(ctx context.Context, r *RootPodReconciler, lr *leafUtils.LeafResource, unstructuredObj *unstructured.Unstructured, rootpod *corev1.Pod, _ *leafUtils.ClusterNode) error } func NewStorageHandler(gvr schema.GroupVersionResource) (StorageHandler, error) { @@ -33,13 +34,26 @@ func NewStorageHandler(gvr schema.GroupVersionResource) (StorageHandler, error) type ConfigMapHandler struct { } -func (c *ConfigMapHandler) BeforeCreateInLeaf(context.Context, *RootPodReconciler, *leafUtils.LeafResource, *unstructured.Unstructured, *corev1.Pod, *leafUtils.ClusterNode) error { +// BeforeGetInLeaf The name of the host cluster kube-root-ca.crt in the leaf group is master-root-ca.crt +func (c *ConfigMapHandler) BeforeGetInLeaf(ctx context.Context, r *RootPodReconciler, lr *leafUtils.LeafResource, unstructuredObj *unstructured.Unstructured, rootpod *corev1.Pod, _ *leafUtils.ClusterNode) error { + if unstructuredObj.GetName() == utils.RooTCAConfigMapName { + unstructuredObj.SetName(utils.MasterRooTCAName) + klog.V(4).Infof("Modify the name of the configmap for the CA: %s", utils.MasterRooTCAName) + } + return nil +} + +func (c *ConfigMapHandler) BeforeCreateInLeaf(ctx context.Context, r *RootPodReconciler, lr *leafUtils.LeafResource, unstructuredObj *unstructured.Unstructured, rootpod *corev1.Pod, _ *leafUtils.ClusterNode) error { return nil } type SecretHandler struct { } +func (s *SecretHandler) BeforeGetInLeaf(ctx context.Context, r *RootPodReconciler, lr *leafUtils.LeafResource, unstructuredObj *unstructured.Unstructured, rootpod *corev1.Pod, _ *leafUtils.ClusterNode) error { + return nil +} + func (s *SecretHandler) BeforeCreateInLeaf(ctx context.Context, r *RootPodReconciler, lr *leafUtils.LeafResource, unstructuredObj *unstructured.Unstructured, rootpod *corev1.Pod, _ *leafUtils.ClusterNode) error { secretObj := &corev1.Secret{} err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredObj.Object, secretObj) @@ -58,6 +72,10 @@ func (s *SecretHandler) BeforeCreateInLeaf(ctx context.Context, r *RootPodReconc type PVCHandler struct { } +func (v *PVCHandler) BeforeGetInLeaf(_ context.Context, _ *RootPodReconciler, lr *leafUtils.LeafResource, unstructuredObj *unstructured.Unstructured, rootpod *corev1.Pod, cn *leafUtils.ClusterNode) error { + return nil +} + func (v *PVCHandler) BeforeCreateInLeaf(_ context.Context, _ *RootPodReconciler, lr *leafUtils.LeafResource, unstructuredObj *unstructured.Unstructured, rootpod *corev1.Pod, cn *leafUtils.ClusterNode) error { if rootpod == nil || len(rootpod.Spec.NodeName) == 0 { return nil diff --git a/pkg/utils/constants.go b/pkg/utils/constants.go index da08a63db..d59db18ac 100644 --- a/pkg/utils/constants.go +++ b/pkg/utils/constants.go @@ -40,6 +40,10 @@ const ( DefaultContainerdSockAddress = "/run/containerd/containerd.sock" DefaultVersion = "latest" DefaultTarName = "kosmos-io.tar.gz" + // nolint + DefaultServiceAccountName = "default" + // nolint + DefaultServiceAccountToken = "kosmos.io/service-account.name" ) const ( diff --git a/pkg/utils/k8s.go b/pkg/utils/k8s.go index f22024a25..e5019bb4b 100644 --- a/pkg/utils/k8s.go +++ b/pkg/utils/k8s.go @@ -246,7 +246,12 @@ func UpdateSecret(old, new *corev1.Secret) { old.Labels = new.Labels old.Data = new.Data old.StringData = new.StringData - old.Type = new.Type + // The satoken type of default in a subset group is Opaque + if old.Annotations[corev1.ServiceAccountNameKey] == DefaultServiceAccountName { + old.Type = corev1.SecretTypeOpaque + } else { + old.Type = new.Type + } } func UpdateUnstructured[T *corev1.ConfigMap | *corev1.Secret](old, new *unstructured.Unstructured, oldObj T, newObj T, update func(old, new T)) (*unstructured.Unstructured, error) { diff --git a/pkg/utils/podutils/pod.go b/pkg/utils/podutils/pod.go index 3044d3387..34636caae 100644 --- a/pkg/utils/podutils/pod.go +++ b/pkg/utils/podutils/pod.go @@ -21,12 +21,8 @@ func GetSecrets(pod *corev1.Pod) ([]string, []string) { for _, v := range pod.Spec.Volumes { switch { case v.Secret != nil: - if strings.HasPrefix(v.Name, "default-token") { - continue - } klog.Infof("pod %s depends on secret %s", pod.Name, v.Secret.SecretName) secretNames = append(secretNames, v.Secret.SecretName) - case v.CephFS != nil: klog.Infof("pod %s depends on secret %s", pod.Name, v.CephFS.SecretRef.Name) secretNames = append(secretNames, v.CephFS.SecretRef.Name) @@ -51,11 +47,16 @@ func GetSecrets(pod *corev1.Pod) ([]string, []string) { func GetConfigmaps(pod *corev1.Pod) []string { cmNames := []string{} - for _, v := range pod.Spec.Volumes { + for i, v := range pod.Spec.Volumes { if v.ConfigMap == nil { continue } cmNames = append(cmNames, v.ConfigMap.Name) + // change the ca.crt mount name of the leaf cluster + // to the ca.crt file of the host cluster that we created + if v.ConfigMap.Name == utils.RooTCAConfigMapName { + pod.Spec.Volumes[i].ConfigMap.Name = utils.MasterRooTCAName + } } klog.Infof("pod %s depends on configMap %s", pod.Name, cmNames) return cmNames @@ -129,6 +130,8 @@ func FitObjectMeta(meta *metav1.ObjectMeta) { meta.OwnerReferences = nil } +// FitUnstructuredObjMeta Leave the version number and OwnerReferences blank +// because kosmos are delivered as Pods (not deploy, etc.) func FitUnstructuredObjMeta(unstructuredObj *unstructured.Unstructured) { unstructuredObj.SetUID("") unstructuredObj.SetResourceVersion("")