Skip to content

Commit

Permalink
1
Browse files Browse the repository at this point in the history 
Signed-off-by: renxiangyu <[email protected]>
  • Loading branch information
@rxy0210
rxy0210 committed Dec 29, 2023
1 parent 9a5878a commit 33e8a4f
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 21 deletions.
85 changes: 66 additions & 19 deletions pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,16 +303,19 @@ func (r *RootPodReconciler) createStorageInLeafCluster(ctx context.Context, lr *
}

// create resource in leaf cluster
_, err = lr.DynamicClient.Resource(gvr).Namespace(ns).Get(ctx, rname, metav1.GetOptions{})
unstructuredObj := rootobj

podutils.FitUnstructuredObjMeta(unstructuredObj)
if err := storageHandler.BeforeGetInLeaf(ctx, r, lr, unstructuredObj, rootpod, cn); err != nil {
return err
}

_, err = lr.DynamicClient.Resource(gvr).Namespace(ns).Get(ctx, unstructuredObj.GetName(), metav1.GetOptions{})
if err == nil {
// already existed, so skip
continue
}
if errors.IsNotFound(err) {
unstructuredObj := rootobj

podutils.FitUnstructuredObjMeta(unstructuredObj)

if err := storageHandler.BeforeCreateInLeaf(ctx, r, lr, unstructuredObj, rootpod, cn); err != nil {
return err
}
Expand Down Expand Up @@ -374,15 +377,25 @@ func (r *RootPodReconciler) createSATokenInLeafCluster(ctx context.Context, lr *
return clientSecret.Name, nil
}

ch := make(chan string, 1)
clusterNodeInfo := r.GlobalLeafManager.GetClusterNode(pod.Spec.NodeName)
if err = wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) {
if err := r.createStorageInLeafCluster(ctx, lr, utils.GVR_SECRET, []string{rootSecretName}, pod, clusterNodeInfo); err == nil {
return true, nil
} else {
return false, err
go func() {
if err = wait.PollImmediate(1*time.Second, 30*time.Second, func() (bool, error) {
if err := r.createStorageInLeafCluster(ctx, lr, utils.GVR_SECRET, []string{rootSecretName}, pod, clusterNodeInfo); err == nil {
return true, nil
} else {
return false, err
}
}); err != nil {
ch <- fmt.Sprintf("could not create secret token %s in leaf cluster: %v", rootSecretName, err)
}
}); err != nil {
return "", fmt.Errorf("could not create secret token %s in leaf cluster: %v", rootSecretName, err)
ch <- ""
}()

t := <-ch
errString := "" + t
if len(errString) > 0 {
return "", fmt.Errorf("%s", errString)
}

return rootSecretName, nil
Expand Down Expand Up @@ -445,9 +458,25 @@ func (r *RootPodReconciler) createConfigMapInLeafCluster(ctx context.Context, lr
return memberConfigMap.Name, nil
}

ch := make(chan string, 1)
clusterNodeInfo := r.GlobalLeafManager.GetClusterNode(pod.Spec.NodeName)
if err = r.createStorageInLeafCluster(ctx, lr, utils.GVR_CONFIGMAP, []string{configMapName}, pod, clusterNodeInfo); err != nil {
return "", fmt.Errorf("could not create configmap %s in member cluster: %v", configMapName, err)
go func() {
if err = wait.PollImmediate(1*time.Second, 30*time.Second, func() (bool, error) {
if err = r.createStorageInLeafCluster(ctx, lr, utils.GVR_CONFIGMAP, []string{configMapName}, pod, clusterNodeInfo); err == nil {
return true, nil
} else {
return false, err
}
}); err != nil {
ch <- fmt.Sprintf("could not create configmap %s in member cluster: %v", configMapName, err)
}
ch <- ""
}()

t := <-ch
errString := "" + t
if len(errString) > 0 {
return "", fmt.Errorf("%s", errString)
}

return memberConfigmapKeyName, nil
Expand All @@ -470,9 +499,25 @@ func (r *RootPodReconciler) createSecretInLeafCluster(ctx context.Context, lr *l
return memberSecret.Name, nil
}

ch := make(chan string, 1)
clusterNodeInfo := r.GlobalLeafManager.GetClusterNode(pod.Spec.NodeName)
if err = r.createStorageInLeafCluster(ctx, lr, utils.GVR_SECRET, []string{secretName}, pod, clusterNodeInfo); err != nil {
return "", fmt.Errorf("could not create secret %s in member cluster: %v", secretName, err)
go func() {
if err = wait.PollImmediate(1*time.Second, 30*time.Second, func() (bool, error) {
if err = r.createStorageInLeafCluster(ctx, lr, utils.GVR_SECRET, []string{secretName}, pod, clusterNodeInfo); err == nil {
return true, nil
} else {
return false, err
}
}); err != nil {
ch <- fmt.Sprintf("could not create secret %s in member cluster: %v", secretName, err)
}
ch <- ""
}()

t := <-ch
errString := "" + t
if len(errString) > 0 {
return "", fmt.Errorf("%s", errString)
}

return secretName, nil
Expand Down Expand Up @@ -526,7 +571,7 @@ func (r *RootPodReconciler) projectedHandler(ctx context.Context, lr *leafUtils.
if projectedVolumeSource.ServiceAccountToken != nil {
tokenSecretName, err := r.createSATokenInLeafCluster(ctx, lr, saName, pod)
if err != nil {
klog.Errorf("[convertAuth] create sa secret failed, ns: %s, pod: %s", pod.Namespace, pod.Name)
klog.Errorf("[convertAuth] create sa secret failed, ns: %s, pod: %s, err: %s", pod.Namespace, pod.Name, err)
return
}
secretProjection := corev1.VolumeProjection{
Expand All @@ -545,7 +590,7 @@ func (r *RootPodReconciler) projectedHandler(ctx context.Context, lr *leafUtils.
if projectedVolumeSource.ConfigMap != nil {
cmName, err := r.createConfigMapInLeafCluster(ctx, lr, projectedVolumeSource.ConfigMap.Name, pod)
if err != nil {
klog.Errorf("[convertAuth] create configmap failed, ns: %s, cm: %s", pod.Namespace, cmName)
klog.Errorf("[convertAuth] create configmap failed, ns: %s, cm: %s, err: %s", pod.Namespace, cmName, err)
return
}
cmDeepCopy := projectedVolumeSource.DeepCopy()
Expand All @@ -556,7 +601,7 @@ func (r *RootPodReconciler) projectedHandler(ctx context.Context, lr *leafUtils.
Secret := projectedVolumeSource.Secret
seName, err := r.createSecretInLeafCluster(ctx, lr, Secret.Name, pod)
if err != nil {
klog.Errorf("[convertAuth] create secret failed, ns: %s, cm: %s", pod.Namespace, seName)
klog.Errorf("[convertAuth] create secret failed, ns: %s, cm: %s, err: %s", pod.Namespace, seName, err)
return
}
secretDeepCopy := projectedVolumeSource.DeepCopy()
Expand Down Expand Up @@ -626,8 +671,10 @@ func (r *RootPodReconciler) createServiceAccountInLeafCluster(ctx context.Contex
secret.ObjectMeta.Namespace = ns
} else {
secret.Annotations[utils.DefaultServiceAccountToken] = utils.DefaultServiceAccountName
secret.Type = corev1.SecretTypeOpaque
}

secret.Annotations[utils.KosmosGlobalLabel] = "true"
err := lr.Client.Create(ctx, secret)

if err != nil {
Expand Down
13 changes: 13 additions & 0 deletions pkg/clustertree/cluster-manager/controllers/pod/storage_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (

type StorageHandler interface {
BeforeCreateInLeaf(context.Context, *RootPodReconciler, *leafUtils.LeafResource, *unstructured.Unstructured, *corev1.Pod, *leafUtils.ClusterNode) error
BeforeGetInLeaf(ctx context.Context, r *RootPodReconciler, lr *leafUtils.LeafResource, unstructuredObj *unstructured.Unstructured, rootpod *corev1.Pod, _ *leafUtils.ClusterNode) error
}

func NewStorageHandler(gvr schema.GroupVersionResource) (StorageHandler, error) {
Expand All @@ -33,6 +34,10 @@ type ConfigMapHandler struct {
}

func (c *ConfigMapHandler) BeforeCreateInLeaf(ctx context.Context, r *RootPodReconciler, lr *leafUtils.LeafResource, unstructuredObj *unstructured.Unstructured, rootpod *corev1.Pod, _ *leafUtils.ClusterNode) error {
return nil
}

func (c *ConfigMapHandler) BeforeGetInLeaf(ctx context.Context, r *RootPodReconciler, lr *leafUtils.LeafResource, unstructuredObj *unstructured.Unstructured, rootpod *corev1.Pod, _ *leafUtils.ClusterNode) error {
if unstructuredObj.GetName() == utils.RooTCAConfigMapName {
unstructuredObj.SetName(utils.MasterRooTCAName)
klog.V(4).Infof("Modify the name of the configmap for the CA: ", utils.MasterRooTCAName)
Expand All @@ -59,6 +64,10 @@ func (s *SecretHandler) BeforeCreateInLeaf(ctx context.Context, r *RootPodReconc
return nil
}

func (s *SecretHandler) BeforeGetInLeaf(ctx context.Context, r *RootPodReconciler, lr *leafUtils.LeafResource, unstructuredObj *unstructured.Unstructured, rootpod *corev1.Pod, _ *leafUtils.ClusterNode) error {
return nil
}

type PVCHandler struct {
}

Expand All @@ -79,3 +88,7 @@ func (v *PVCHandler) BeforeCreateInLeaf(_ context.Context, _ *RootPodReconciler,

return nil
}

func (v *PVCHandler) BeforeGetInLeaf(_ context.Context, _ *RootPodReconciler, lr *leafUtils.LeafResource, unstructuredObj *unstructured.Unstructured, rootpod *corev1.Pod, cn *leafUtils.ClusterNode) error {
return nil
}
6 changes: 5 additions & 1 deletion pkg/utils/k8s.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,11 @@ func UpdateSecret(old, new *corev1.Secret) {
old.Labels = new.Labels
old.Data = new.Data
old.StringData = new.StringData
old.Type = new.Type
if old.Annotations[corev1.ServiceAccountNameKey] == DefaultServiceAccountName {
old.Type = corev1.SecretTypeOpaque
} else {
old.Type = new.Type
}
}

func UpdateUnstructured[T *corev1.ConfigMap | *corev1.Secret](old, new *unstructured.Unstructured, oldObj T, newObj T, update func(old, new T)) (*unstructured.Unstructured, error) {
Expand Down
5 changes: 4 additions & 1 deletion pkg/utils/podutils/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,14 @@ func GetSecrets(pod *corev1.Pod) ([]string, []string) {

func GetConfigmaps(pod *corev1.Pod) []string {
cmNames := []string{}
for _, v := range pod.Spec.Volumes {
for i, v := range pod.Spec.Volumes {
if v.ConfigMap == nil {
continue
}
cmNames = append(cmNames, v.ConfigMap.Name)
if v.ConfigMap.Name == utils.RooTCAConfigMapName {
pod.Spec.Volumes[i].ConfigMap.Name = utils.MasterRooTCAName
}
}
klog.Infof("pod %s depends on configMap %s", pod.Name, cmNames)
return cmNames
Expand Down

0 comments on commit 33e8a4f

Please sign in to comment.