diff --git a/deploy/crds/kosmos.io_podconvertpolicies.yaml b/deploy/crds/kosmos.io_podconvertpolicies.yaml index 1010ffffc..108b6a149 100644 --- a/deploy/crds/kosmos.io_podconvertpolicies.yaml +++ b/deploy/crds/kosmos.io_podconvertpolicies.yaml @@ -1012,6 +1012,62 @@ spec: required: - convertType type: object + tolerationConverter: + description: TolerationConverter used to modify the pod's Tolerations + when pod synced to leaf cluster + properties: + convertType: + description: ConvertType if the operation type when convert + pod from root cluster to leaf cluster. + enum: + - add + - remove + - replace + type: string + tolerations: + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, + allowed values are NoSchedule, PreferNoSchedule and + NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. If the + key is empty, operator must be Exists; this combination + means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and Equal. + Defaults to Equal. Exists is equivalent to wildcard + for value, so that a pod can tolerate all taints of + a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period + of time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the + taint forever (do not evict). Zero and negative values + will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration + matches to. If the operator is Exists, the value should + be empty, otherwise just a regular string. + type: string + type: object + type: array + required: + - convertType + type: object topologySpreadConstraintsConverter: description: TopologySpreadConstraintsConverter used to modify the pod's TopologySpreadConstraints when pod synced to leaf diff --git a/pkg/apis/kosmos/v1alpha1/podconvertpolicy_types.go b/pkg/apis/kosmos/v1alpha1/podconvertpolicy_types.go index e1da2a0aa..d597f4f06 100644 --- a/pkg/apis/kosmos/v1alpha1/podconvertpolicy_types.go +++ b/pkg/apis/kosmos/v1alpha1/podconvertpolicy_types.go @@ -45,6 +45,8 @@ type Converters struct { // +optional NodeSelectorConverter *NodeSelectorConverter `json:"nodeSelectorConverter,omitempty"` // +optional + TolerationConverter *TolerationConverter `json:"tolerationConverter,omitempty"` + // +optional AffinityConverter *AffinityConverter `json:"affinityConverter,omitempty"` // +optional TopologySpreadConstraintsConverter *TopologySpreadConstraintsConverter `json:"topologySpreadConstraintsConverter,omitempty"` diff --git a/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go index 181799bf6..6ba746a20 100644 --- a/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go @@ -382,6 +382,11 @@ func (in *Converters) DeepCopyInto(out *Converters) { *out = new(NodeSelectorConverter) (*in).DeepCopyInto(*out) } + if in.TolerationConverter != nil { + in, out := &in.TolerationConverter, &out.TolerationConverter + *out = new(TolerationConverter) + (*in).DeepCopyInto(*out) + } if in.AffinityConverter != nil { in, out := &in.AffinityConverter, &out.AffinityConverter *out = new(AffinityConverter) diff --git a/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go b/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go index 9fea35173..c71f65e44 100644 --- a/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go +++ b/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go @@ -31,6 +31,7 @@ import ( "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/extensions/daemonset" leafUtils "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/utils" "github.com/kosmos.io/kosmos/pkg/utils" + "github.com/kosmos.io/kosmos/pkg/utils/convertpolicy" "github.com/kosmos.io/kosmos/pkg/utils/podutils" ) @@ -702,6 +703,37 @@ func (r *RootPodReconciler) createVolumes(ctx context.Context, lr *leafUtils.Lea return nil } +// mutatePod modify pod by matching policy +func (r *RootPodReconciler) mutatePod(ctx context.Context, pod *corev1.Pod, nodeName string) error { + klog.V(4).Infof("Converting pod %v/%+v", pod.Namespace, pod.Name) + + podConvertPolicyList := &kosmosv1alpha1.PodConvertPolicyList{} + err := r.Client.List(ctx, podConvertPolicyList, &client.ListOptions{ + Namespace: pod.Namespace, + }) + if err != nil { + return fmt.Errorf("list convert policy error: %v", err) + } + if len(podConvertPolicyList.Items) <= 0 { + // no matched policy, skip + return nil + } + + rootNode := &corev1.Node{} + err = r.Client.Get(ctx, types.NamespacedName{Name: nodeName}, rootNode) + if err != nil { + return fmt.Errorf("get node error: %v, nodeName: %s", err, pod.Spec.NodeName) + } + + matchedPolicy, err := convertpolicy.GetMatchPodConvertPolicy(*podConvertPolicyList, pod.Labels, rootNode.Labels) + if err != nil { + return fmt.Errorf("get convert policy error: %v", err) + } + podutils.ConvertPod(pod, matchedPolicy) + klog.V(4).Infof("Convert pod %v/%+v success", pod.Namespace, pod.Name) + return nil +} + func (r *RootPodReconciler) CreatePodInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, pod *corev1.Pod, nodeSelector kosmosv1alpha1.NodeSelector) error { if err := podutils.PopulateEnvironmentVariables(ctx, pod, r.envResourceManager); err != nil { // span.SetStatus(err) @@ -716,6 +748,11 @@ func (r *RootPodReconciler) CreatePodInLeafCluster(ctx context.Context, lr *leaf basicPod := podutils.FitPod(pod, lr.IgnoreLabels, clusterNodeInfo.LeafMode, nodeSelector) klog.V(4).Infof("Creating pod %v/%+v", pod.Namespace, pod.Name) + err := r.mutatePod(ctx, basicPod, pod.Spec.NodeName) + if err != nil { + klog.Errorf("Converting pod error: %v", err) + } + // create ns ns := &corev1.Namespace{} nsKey := types.NamespacedName{ @@ -759,7 +796,7 @@ func (r *RootPodReconciler) CreatePodInLeafCluster(ctx context.Context, lr *leaf klog.V(4).Infof("Creating pod %+v", basicPod) - err := lr.Client.Create(ctx, basicPod) + err = lr.Client.Create(ctx, basicPod) if err != nil { return fmt.Errorf("could not create pod: %v", err) } diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 0d3120e96..8872dc8ca 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -748,6 +748,11 @@ func schema_pkg_apis_kosmos_v1alpha1_Converters(ref common.ReferenceCallback) co Ref: ref("github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.NodeSelectorConverter"), }, }, + "tolerationConverter": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.TolerationConverter"), + }, + }, "affinityConverter": { SchemaProps: spec.SchemaProps{ Ref: ref("github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.AffinityConverter"), @@ -762,7 +767,7 @@ func schema_pkg_apis_kosmos_v1alpha1_Converters(ref common.ReferenceCallback) co }, }, Dependencies: []string{ - "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.AffinityConverter", "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.NodeNameConverter", "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.NodeSelectorConverter", "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.SchedulerNameConverter", "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.TopologySpreadConstraintsConverter"}, + "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.AffinityConverter", "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.NodeNameConverter", "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.NodeSelectorConverter", "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.SchedulerNameConverter", "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.TolerationConverter", "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.TopologySpreadConstraintsConverter"}, } } diff --git a/pkg/utils/constants.go b/pkg/utils/constants.go index 52c36210f..3b78795e4 100644 --- a/pkg/utils/constants.go +++ b/pkg/utils/constants.go @@ -106,6 +106,7 @@ const ( KosmosGlobalLabel = "kosmos.io/global" KosmosSelectorKey = "kosmos.io/cluster-selector" KosmosTrippedLabels = "kosmos-io/tripped" + KosmosConvertLabels = "kosmos-io/convert-policy" KosmosPvcLabelSelector = "kosmos-io/label-selector" KosmosExcludeNodeLabel = "kosmos.io/exclude" KosmosExcludeNodeValue = "true" diff --git a/pkg/utils/convertpolicy/pod.go b/pkg/utils/convertpolicy/pod.go new file mode 100644 index 000000000..968c7e050 --- /dev/null +++ b/pkg/utils/convertpolicy/pod.go @@ -0,0 +1,41 @@ +package convertpolicy + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + kosmosv1alpha1 "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" +) + +// GetMatchPodConvertPolicy returns the PodConvertPolicies matching label selector +func GetMatchPodConvertPolicy(policies kosmosv1alpha1.PodConvertPolicyList, podLabels map[string]string, nodeLabels map[string]string) ([]kosmosv1alpha1.PodConvertPolicy, error) { + matched := make([]kosmosv1alpha1.PodConvertPolicy, 0) + + var podSelector, nodeSelector labels.Selector + var err error + for _, po := range policies.Items { + spec := po.Spec + podSelector, err = metav1.LabelSelectorAsSelector(&spec.LabelSelector) + if err != nil { + return nil, err + } + if !podSelector.Matches(labels.Set(podLabels)) { + continue + } + + if spec.LeafNodeSelector == nil { + // matches all leafNode. + nodeSelector = labels.Everything() + } else { + if nodeSelector, err = metav1.LabelSelectorAsSelector(spec.LeafNodeSelector); err != nil { + return nil, err + } + } + if !nodeSelector.Matches(labels.Set(nodeLabels)) { + continue + } + + matched = append(matched, po) + } + return matched, nil +} diff --git a/pkg/utils/podutils/pod_convert.go b/pkg/utils/podutils/pod_convert.go new file mode 100644 index 000000000..1b16b284f --- /dev/null +++ b/pkg/utils/podutils/pod_convert.go @@ -0,0 +1,153 @@ +package podutils + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/klog" + + kosmosv1alpha1 "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" + "github.com/kosmos.io/kosmos/pkg/utils" +) + +// ConvertPod perform all conversions +func ConvertPod(pod *corev1.Pod, policies []kosmosv1alpha1.PodConvertPolicy) { + if len(policies) <= 0 { + return + } + + var choose *kosmosv1alpha1.PodConvertPolicy + // current, use the first non-empty matching policy + for idx, po := range policies { + if po.Spec.Converters != nil { + choose = &policies[idx] + break + } + } + if choose == nil { + return + } + klog.V(4).Infof("Convert pod %v/%+v, policy: %s", pod.Namespace, pod.Name, choose.Name) + + converters := choose.Spec.Converters + convertSchedulerName(pod, converters.SchedulerNameConverter) + convertNodeName(pod, converters.NodeNameConverter) + convertNodeSelector(pod, converters.NodeSelectorConverter) + converToleration(pod, converters.TolerationConverter) + convertAffinity(pod, converters.AffinityConverter) + convertTopologySpreadConstraints(pod, converters.TopologySpreadConstraintsConverter) + + pod.Annotations[utils.KosmosConvertLabels] = choose.Name +} + +func convertSchedulerName(pod *corev1.Pod, converter *kosmosv1alpha1.SchedulerNameConverter) { + if converter == nil { + return + } + + switch converter.ConvertType { + case kosmosv1alpha1.Add: + if pod.Spec.SchedulerName == "" { + pod.Spec.SchedulerName = converter.SchedulerName + } + case kosmosv1alpha1.Remove: + pod.Spec.SchedulerName = "" + case kosmosv1alpha1.Replace: + pod.Spec.SchedulerName = converter.SchedulerName + default: + klog.Warningf("Skip other convert type, SchedulerName: %s", converter.ConvertType) + } +} + +func convertNodeName(pod *corev1.Pod, converter *kosmosv1alpha1.NodeNameConverter) { + if converter == nil { + return + } + + switch converter.ConvertType { + case kosmosv1alpha1.Add: + if pod.Spec.NodeName == "" { + pod.Spec.NodeName = converter.NodeName + } + case kosmosv1alpha1.Remove: + pod.Spec.NodeName = "" + case kosmosv1alpha1.Replace: + pod.Spec.NodeName = converter.NodeName + default: + klog.Warningf("Skip other convert type, NodeName: %s", converter.ConvertType) + } +} + +func converToleration(pod *corev1.Pod, conveter *kosmosv1alpha1.TolerationConverter) { + if conveter == nil { + return + } + + switch conveter.ConvertType { + case kosmosv1alpha1.Add: + if pod.Spec.Tolerations == nil { + pod.Spec.Tolerations = conveter.Tolerations + } + case kosmosv1alpha1.Remove: + pod.Spec.Tolerations = nil + case kosmosv1alpha1.Replace: + pod.Spec.Tolerations = conveter.Tolerations + default: + klog.Warningf("Skip other convert type, Tolerations: %s", conveter.ConvertType) + } +} + +func convertNodeSelector(pod *corev1.Pod, converter *kosmosv1alpha1.NodeSelectorConverter) { + if converter == nil { + return + } + + switch converter.ConvertType { + case kosmosv1alpha1.Add: + if pod.Spec.NodeSelector == nil { + pod.Spec.NodeSelector = converter.NodeSelector + } + case kosmosv1alpha1.Remove: + pod.Spec.NodeSelector = nil + case kosmosv1alpha1.Replace: + pod.Spec.NodeSelector = converter.NodeSelector + default: + klog.Warningf("Skip other convert type, NodeSelector: %s", converter.ConvertType) + } +} + +func convertAffinity(pod *corev1.Pod, converter *kosmosv1alpha1.AffinityConverter) { + if converter == nil { + return + } + + switch converter.ConvertType { + case kosmosv1alpha1.Add: + if pod.Spec.Affinity == nil { + pod.Spec.Affinity = converter.Affinity + } + case kosmosv1alpha1.Remove: + pod.Spec.Affinity = nil + case kosmosv1alpha1.Replace: + pod.Spec.Affinity = converter.Affinity + default: + klog.Warningf("Skip other convert type, Affinity: %s", converter.ConvertType) + } +} + +func convertTopologySpreadConstraints(pod *corev1.Pod, converter *kosmosv1alpha1.TopologySpreadConstraintsConverter) { + if converter == nil { + return + } + + switch converter.ConvertType { + case kosmosv1alpha1.Add: + if pod.Spec.Affinity == nil { + pod.Spec.TopologySpreadConstraints = converter.TopologySpreadConstraints + } + case kosmosv1alpha1.Remove: + pod.Spec.TopologySpreadConstraints = nil + case kosmosv1alpha1.Replace: + pod.Spec.TopologySpreadConstraints = converter.TopologySpreadConstraints + default: + klog.Warningf("Skip other convert type, TopologySpreadConstraints: %s", converter.ConvertType) + } +}