Skip to content

Commit

Permalink
fix: adapt satoken
Browse files Browse the repository at this point in the history 
Signed-off-by: renxiangyu <[email protected]>
  • Loading branch information
renxiangyu committed Dec 22, 2023
1 parent 04938bf commit 1691f36
Show file tree
Hide file tree
Showing 2 changed files with 207 additions and 63 deletions.
260 changes: 201 additions & 59 deletions pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,15 @@ func (r *RootPodReconciler) createSATokenInLeafCluster(ctx context.Context, lr *
secretName = sa.Secrets[0].Name
}

if secretName == "" {
tokenSecret, err := r.createDefaultSecretInRootCluster(ctx, sa)
if err != nil {
return nil, err
}

secretName = tokenSecret.Name
}

csName := fmt.Sprintf("master-%s-token", sa.Name)
csKey := types.NamespacedName{
Namespace: ns,
Expand Down Expand Up @@ -424,44 +433,122 @@ func (r *RootPodReconciler) createSATokenInLeafCluster(ctx context.Context, lr *
return newSE, nil
}

func (r *RootPodReconciler) createCAInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, ns string) (*corev1.ConfigMap, error) {
masterCAConfigmapKey := types.NamespacedName{
func (r *RootPodReconciler) createDefaultSecretInRootCluster(ctx context.Context, sa *corev1.ServiceAccount) (*corev1.Secret, error) {
tokenSecretName := fmt.Sprintf("kosmos-%s-token", sa.Name)
tokenSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: tokenSecretName,
Namespace: sa.Namespace,
Annotations: map[string]string{
utils.TokenSecretAnnotations: sa.Name,
},
},
Type: utils.TokenSecretType,
}
err := r.RootClient.Create(ctx, tokenSecret)

if err != nil {
return nil, fmt.Errorf("could not create token-secret %s in host cluster: %v", tokenSecretName, err)
}

patchSa := sa.DeepCopy()
patchSa.Secrets = []corev1.ObjectReference{
{
Name: tokenSecretName,
},
}
err = r.RootClient.Update(ctx, patchSa)
if err != nil {
return nil, fmt.Errorf("could not update sa %s in host cluster: %v", patchSa.Name, err)
}

return tokenSecret, nil
}

func (r *RootPodReconciler) createConfigMapInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, configMapName string, ns string) (*corev1.ConfigMap, error) {
var memberConfigmapKeyName string
if strings.HasPrefix(configMapName, utils.RooTCAConfigMapName) {
memberConfigmapKeyName = utils.MasterRooTCAName
} else {
memberConfigmapKeyName = configMapName
}

configmapKey := types.NamespacedName{
Namespace: ns,
Name: utils.MasterRooTCAName,
Name: memberConfigmapKeyName,
}

masterCA := &corev1.ConfigMap{}
memberConfigMap := &corev1.ConfigMap{}

err := lr.Client.Get(ctx, masterCAConfigmapKey, masterCA)
err := lr.Client.Get(ctx, configmapKey, memberConfigMap)
if err != nil && !errors.IsNotFound(err) {
return nil, fmt.Errorf("could not check configmap %s in member cluster: %v", utils.MasterRooTCAName, err)
return nil, fmt.Errorf("could not check configmap %s in member cluster: %v", configmapKey.Name, err)
}
if err == nil {
return masterCA, nil
return memberConfigMap, nil
}

ca := &corev1.ConfigMap{}
masterConfigMap := &corev1.ConfigMap{}

rootCAConfigmapKey := types.NamespacedName{
masterConfigmapKey := types.NamespacedName{
Namespace: ns,
Name: utils.RooTCAConfigMapName,
Name: configMapName,
}

err = r.Client.Get(ctx, rootCAConfigmapKey, ca)
err = r.Client.Get(ctx, masterConfigmapKey, masterConfigMap)
if err != nil {
return nil, fmt.Errorf("could not find configmap %s in master cluster: %v", ca, err)
return nil, fmt.Errorf("could not find configmap %s in master cluster: %v", configmapKey.Name, err)
}

newCA := ca.DeepCopy()
newCA.Name = utils.MasterRooTCAName
podutils.FitObjectMeta(&newCA.ObjectMeta)
newConfigMap := masterConfigMap.DeepCopy()
newConfigMap.Name = memberConfigmapKeyName
podutils.FitObjectMeta(&newConfigMap.ObjectMeta)

err = lr.Client.Create(ctx, newCA)
err = lr.Client.Create(ctx, newConfigMap)
if err != nil && !errors.IsAlreadyExists(err) {
return nil, fmt.Errorf("could not create configmap %s in member cluster: %v", newCA.Name, err)
return nil, fmt.Errorf("could not create configmap %s in member cluster: %v", newConfigMap.Name, err)
}

return newCA, nil
return newConfigMap, nil
}

func (r *RootPodReconciler) createSecretInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, secretName string, ns string) (*corev1.Secret, error) {
secretKey := types.NamespacedName{
Namespace: ns,
Name: secretName,
}

memberSecret := &corev1.Secret{}

err := lr.Client.Get(ctx, secretKey, memberSecret)
if err != nil && !errors.IsNotFound(err) {
return nil, fmt.Errorf("could not check secret %s in member cluster: %v", secretKey.Name, err)
}
if err == nil {
return memberSecret, nil
}

masterSecret := &corev1.Secret{}

masterSecretKey := types.NamespacedName{
Namespace: ns,
Name: secretName,
}

err = r.Client.Get(ctx, masterSecretKey, masterSecret)
if err != nil {
return nil, fmt.Errorf("could not find secret %s in master cluster: %v", masterSecretKey.Name, err)
}

newSecret := masterSecret.DeepCopy()
podutils.FitObjectMeta(&newSecret.ObjectMeta)

err = lr.Client.Create(ctx, newSecret)
if err != nil && !errors.IsAlreadyExists(err) {
return nil, fmt.Errorf("could not create secret %s in member cluster: %v", newSecret.Name, err)
}

return newSecret, nil
}

// changeToMasterCoreDNS point the dns of the pod to the master cluster, so that the pod can access any service.
Expand Down Expand Up @@ -496,61 +583,116 @@ func (r *RootPodReconciler) changeToMasterCoreDNS(ctx context.Context, pod *core
}

func (r *RootPodReconciler) convertAuth(ctx context.Context, lr *leafUtils.LeafResource, pod *corev1.Pod) {
if pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken {
falseValue := false
pod.Spec.AutomountServiceAccountToken = &falseValue
if pod.Spec.Volumes == nil {
return
}

sa := pod.Spec.ServiceAccountName
_, err := r.createSAInLeafCluster(ctx, lr, sa, pod.Namespace)
if err != nil {
klog.Errorf("[convertAuth] create sa failed, ns: %s, pod: %s", pod.Namespace, pod.Name)
return
}
for _, volume := range pod.Spec.Volumes {
if volume.Projected != nil {
falseValue := false
pod.Spec.AutomountServiceAccountToken = &falseValue

se, err := r.createSATokenInLeafCluster(ctx, lr, sa, pod.Namespace)
if err != nil {
klog.Errorf("[convertAuth] create sa secret failed, ns: %s, pod: %s", pod.Namespace, pod.Name)
return
}
saName := pod.Spec.ServiceAccountName
_, err := r.createSAInLeafCluster(ctx, lr, saName, pod.Namespace)
if err != nil {
klog.Errorf("[convertAuth] create sa failed, ns: %s, pod: %s", pod.Namespace, pod.Name)
return
}

rootCA, err := r.createCAInLeafCluster(ctx, lr, pod.Namespace)
if err != nil {
klog.Errorf("[convertAuth] create sa secret failed, ns: %s, pod: %s", pod.Namespace, pod.Name)
return
}
sources := []corev1.VolumeProjection{}

volumes := pod.Spec.Volumes
for _, v := range volumes {
if strings.HasPrefix(v.Name, utils.SATokenPrefix) {
sources := []corev1.VolumeProjection{}
for _, src := range v.Projected.Sources {
if src.ServiceAccountToken != nil {
continue
for _, projectedVolumeSource := range volume.Projected.Sources {
if projectedVolumeSource.ServiceAccountToken != nil {
serviceAccountToken := projectedVolumeSource.ServiceAccountToken
tokenSecret, err := r.createSATokenInLeafCluster(ctx, lr, saName, pod.Namespace)
if err != nil {
klog.Errorf("[convertAuth] create sa secret failed, ns: %s, pod: %s", pod.Namespace, pod.Name)
return
}
if src.ConfigMap != nil && src.ConfigMap.Name == utils.RooTCAConfigMapName {
src.ConfigMap.Name = rootCA.Name
secretProjection := corev1.VolumeProjection{
Secret: &corev1.SecretProjection{
Items: []corev1.KeyToPath{
{
Key: "token",
Path: serviceAccountToken.Path,
},
},
},
}
sources = append(sources, src)
secretProjection.Secret.Name = tokenSecret.Name
sources = append(sources, secretProjection)
}

secretProjection := corev1.VolumeProjection{
Secret: &corev1.SecretProjection{
Items: []corev1.KeyToPath{
{
Key: "token",
Path: "token",
if projectedVolumeSource.ConfigMap != nil {
configMap := projectedVolumeSource.ConfigMap
cm, err := r.createConfigMapInLeafCluster(ctx, lr, configMap.Name, pod.Namespace)
if err != nil {
klog.Errorf("[convertAuth] create configmap failed, ns: %s, cm: %s", pod.Namespace, cm.Name)
return
}
var key string
var path string
for _, item := range configMap.Items {
if item.Key != "" {
key = item.Key
}
if item.Path != "" {
path = item.Path
}
}
configMapProjection := corev1.VolumeProjection{
ConfigMap: &corev1.ConfigMapProjection{
Items: []corev1.KeyToPath{
{
Key: key,
Path: path,
},
},
},
},
}
configMapProjection.ConfigMap.Name = cm.Name
sources = append(sources, configMapProjection)
}
if projectedVolumeSource.Secret != nil {
Secret := projectedVolumeSource.Secret
se, err := r.createSecretInLeafCluster(ctx, lr, Secret.Name, pod.Namespace)
if err != nil {
klog.Errorf("[convertAuth] create secret failed, ns: %s, cm: %s", pod.Namespace, se.Name)
return
}
var key string
var path string
for _, item := range Secret.Items {
if item.Key != "" {
key = item.Key
}
if item.Path != "" {
path = item.Path
}
}
SecretProjection := corev1.VolumeProjection{
Secret: &corev1.SecretProjection{
Items: []corev1.KeyToPath{
{
Key: key,
Path: path,
},
},
},
}
SecretProjection.Secret.Name = se.Name
sources = append(sources, SecretProjection)
}
if projectedVolumeSource.DownwardAPI != nil {
DownwardAPIProjection := corev1.VolumeProjection{
DownwardAPI: projectedVolumeSource.DownwardAPI,
}
sources = append(sources, DownwardAPIProjection)
}
secretProjection.Secret.Name = se.Name
sources = append(sources, secretProjection)
v.Projected.Sources = sources
}
volume.Projected.Sources = sources
}
}
}

func (r *RootPodReconciler) createServiceAccountInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, secret *corev1.Secret) error {
if !lr.EnableServiceAccount {
return nil
Expand Down
10 changes: 6 additions & 4 deletions pkg/utils/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,10 +156,12 @@ const (
)

const (
ReservedNS = "kube-system"
RooTCAConfigMapName = "kube-root-ca.crt"
SATokenPrefix = "kube-api-access"
MasterRooTCAName = "master-root-ca.crt"
ReservedNS = "kube-system"
RooTCAConfigMapName = "kube-root-ca.crt"
SATokenPrefix = "kube-api-access"
MasterRooTCAName = "master-root-ca.crt"
TokenSecretType = "kubernetes.io/service-account-token"

Check failure on line 163 in pkg/utils/constants.go

View workflow job for this annotation

GitHub Actions / lint 

G101: Potential hardcoded credentials (gosec)

Check failure on line 163 in pkg/utils/constants.go

View workflow job for this annotation

GitHub Actions / lint 

G101: Potential hardcoded credentials (gosec)
TokenSecretAnnotations = "kubernetes.io/service-account.name"

Check failure on line 164 in pkg/utils/constants.go

View workflow job for this annotation

GitHub Actions / lint 

G101: Potential hardcoded credentials (gosec)

Check failure on line 164 in pkg/utils/constants.go

View workflow job for this annotation

GitHub Actions / lint 

G101: Potential hardcoded credentials (gosec)
)

var GVR_CONFIGMAP = schema.GroupVersionResource{
Expand Down

0 comments on commit 1691f36

Please sign in to comment.