diff --git a/deploy/crds/kosmos.io_clusters.yaml b/deploy/crds/kosmos.io_clusters.yaml index 9b887b8a9..eea5c4974 100644 --- a/deploy/crds/kosmos.io_clusters.yaml +++ b/deploy/crds/kosmos.io_clusters.yaml @@ -115,59 +115,10 @@ spec: default: true type: boolean leafModel: - description: LeafModel provide an api to arrange the member cluster + description: LeafModels provide an api to arrange the member cluster with some rules to pretend one or more leaf node items: properties: - labelSelector: - description: LabelSelector is a filter to select member - cluster nodes to pretend a leaf node in clusterTree by - labels. If nil or empty, the hole member cluster nodes - will pretend one leaf node. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector - that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, - NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. - If the operator is In or NotIn, the values array - must be non-empty. If the operator is Exists - or DoesNotExist, the values array must be empty. - This array is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. - A single {key,value} in the matchLabels map is equivalent - to an element of matchExpressions, whose key field - is "key", the operator is "In", and the values array - contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic labels: additionalProperties: type: string @@ -179,6 +130,65 @@ spec: the leaf node name will generate by controller and fill in cluster link status type: string + nodeSelector: + description: NodeSelector is a selector to select member + cluster nodes to pretend a leaf node in clusterTree. + properties: + labelSelector: + description: LabelSelector is a filter to select member + cluster nodes to pretend a leaf node in clusterTree + by labels. It will work on second level schedule on + pod create in member clusters. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + nodeName: + description: NodeName is Member cluster origin node + Name + type: string + type: object taints: description: Taints attached to the leaf pretended Node. If nil or empty, controller will set the default no-schedule diff --git a/deploy/crds/kosmos.io_knodes.yaml b/deploy/crds/kosmos.io_knodes.yaml new file mode 100644 index 000000000..647c18313 --- /dev/null +++ b/deploy/crds/kosmos.io_knodes.yaml @@ -0,0 +1,127 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.0 + creationTimestamp: null + name: knodes.kosmos.io +spec: + group: kosmos.io + names: + kind: Knode + listKind: KnodeList + plural: knodes + singular: knode + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + disableTaint: + type: boolean + kubeAPIBurst: + default: 100 + type: integer + kubeconfig: + format: byte + type: string + nodeName: + type: string + type: + default: k8s + type: string + type: object + status: + properties: + apiserver: + type: string + conditions: + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + version: + type: string + type: object + type: object + served: true + storage: true diff --git a/pkg/apis/kosmos/v1alpha1/cluster_types.go b/pkg/apis/kosmos/v1alpha1/cluster_types.go index 053301bb5..51c55a29a 100644 --- a/pkg/apis/kosmos/v1alpha1/cluster_types.go +++ b/pkg/apis/kosmos/v1alpha1/cluster_types.go @@ -98,10 +98,9 @@ type ClusterTreeOptions struct { // +optional Enable bool `json:"enable"` - // LeafModel provide an api to arrange the member cluster with some rules to pretend one or more leaf node - // If nil or empty, the hole member cluster nodes will pretend one leaf node. + // LeafModels provide an api to arrange the member cluster with some rules to pretend one or more leaf node // +optional - LeafModel []LeafModel `json:"leafModel,omitempty"` + LeafModels []LeafModel `json:"leafModel,omitempty"` } type LeafModel struct { diff --git a/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go index 3c45fc1db..d2c5c55de 100644 --- a/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go @@ -294,8 +294,8 @@ func (in *ClusterStatus) DeepCopy() *ClusterStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterTreeOptions) DeepCopyInto(out *ClusterTreeOptions) { *out = *in - if in.LeafModel != nil { - in, out := &in.LeafModel, &out.LeafModel + if in.LeafModels != nil { + in, out := &in.LeafModels, &out.LeafModels *out = make([]LeafModel, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) diff --git a/pkg/clustertree/cluster-manager/cluster_controller.go b/pkg/clustertree/cluster-manager/cluster_controller.go index 96306eac9..d03b5adbe 100644 --- a/pkg/clustertree/cluster-manager/cluster_controller.go +++ b/pkg/clustertree/cluster-manager/cluster_controller.go @@ -172,12 +172,14 @@ func (c *ClusterController) Reconcile(ctx context.Context, request reconcile.Req return reconcile.Result{}, nil } - node, err := c.createNode(ctx, cluster, leafClient) + nodes, err := c.createNode(ctx, cluster, leafClient) if err != nil { return reconcile.Result{}, fmt.Errorf("create node with err %v, cluster %s", err, cluster.Name) } // TODO @wyz - node.ResourceVersion = "" + for _, node := range nodes { + node.ResourceVersion = "" + } // build mgr for cluster // TODO bug, the v4 log is lost @@ -199,7 +201,7 @@ func (c *ClusterController) Reconcile(ctx context.Context, request reconcile.Req c.ManagerCancelFuncs[cluster.Name] = &cancel c.ControllerManagersLock.Unlock() - if err = c.setupControllers(mgr, cluster, node, leafDynamic, leafClient, kosmosClient); err != nil { + if err = c.setupControllers(mgr, cluster, nodes, leafDynamic, leafClient, kosmosClient); err != nil { return reconcile.Result{}, fmt.Errorf("failed to setup cluster %s controllers: %v", cluster.Name, err) } @@ -228,14 +230,14 @@ func (c *ClusterController) clearClusterControllers(cluster *kosmosv1alpha1.Clus c.GlobalLeafManager.RemoveLeafResource(cluster.Name) } -func (c *ClusterController) setupControllers(mgr manager.Manager, cluster *kosmosv1alpha1.Cluster, node *corev1.Node, clientDynamic *dynamic.DynamicClient, leafClient kubernetes.Interface, kosmosClient kosmosversioned.Interface) error { - nodeName := fmt.Sprintf("%s%s", utils.KosmosNodePrefix, cluster.Name) - c.GlobalLeafManager.AddLeafResource(nodeName, &leafUtils.LeafResource{ +func (c *ClusterController) setupControllers(mgr manager.Manager, cluster *kosmosv1alpha1.Cluster, nodes []*corev1.Node, clientDynamic *dynamic.DynamicClient, leafClient kubernetes.Interface, kosmosClient kosmosversioned.Interface) error { + clusterName := fmt.Sprintf("%s%s", utils.KosmosNodePrefix, cluster.Name) + c.GlobalLeafManager.AddLeafResource(clusterName, &leafUtils.LeafResource{ Client: mgr.GetClient(), DynamicClient: clientDynamic, Clientset: leafClient, KosmosClient: kosmosClient, - NodeName: nodeName, + NodeName: clusterName, Namespace: "", IgnoreLabels: strings.Split("", ","), EnableServiceAccount: true, @@ -245,13 +247,14 @@ func (c *ClusterController) setupControllers(mgr manager.Manager, cluster *kosmo Leaf: mgr.GetClient(), Root: c.Root, RootClientset: c.RootClient, - Node: node, + Nodes: nodes, + Cluster: cluster, } if err := nodeResourcesController.SetupWithManager(mgr); err != nil { return fmt.Errorf("error starting %s: %v", controllers.NodeResourcesControllerName, err) } - nodeLeaseController := controllers.NewNodeLeaseController(leafClient, c.Root, node, c.RootClient) + nodeLeaseController := controllers.NewNodeLeaseController(leafClient, c.Root, nodes, c.RootClient) if err := mgr.Add(nodeLeaseController); err != nil { return fmt.Errorf("error starting %s: %v", controllers.NodeLeaseControllerName, err) } @@ -262,7 +265,7 @@ func (c *ClusterController) setupControllers(mgr manager.Manager, cluster *kosmo RootKosmosClient: kosmosClient, EventRecorder: mgr.GetEventRecorderFor(mcs.LeafServiceImportControllerName), Logger: mgr.GetLogger(), - LeafNodeName: nodeName, + LeafNodeName: clusterName, RootResourceManager: c.RootResourceManager, } if err := serviceImportController.AddController(mgr); err != nil { @@ -279,7 +282,7 @@ func (c *ClusterController) setupControllers(mgr manager.Manager, cluster *kosmo return fmt.Errorf("error starting podUpstreamReconciler %s: %v", podcontrollers.LeafPodControllerName, err) } - err := c.setupStorageControllers(mgr, node, leafClient) + err := c.setupStorageControllers(mgr, nodes, leafClient) if err != nil { return err } @@ -287,12 +290,12 @@ func (c *ClusterController) setupControllers(mgr manager.Manager, cluster *kosmo return nil } -func (c *ClusterController) setupStorageControllers(mgr manager.Manager, node *corev1.Node, leafClient kubernetes.Interface) error { +func (c *ClusterController) setupStorageControllers(mgr manager.Manager, nodes []*corev1.Node, leafClient kubernetes.Interface) error { leafPVCController := pvc.LeafPVCController{ LeafClient: mgr.GetClient(), RootClient: c.Root, RootClientSet: c.RootClient, - NodeName: node.Name, + NodeName: nodes[0].Name, } if err := leafPVCController.SetupWithManager(mgr); err != nil { return fmt.Errorf("error starting leaf pvc controller %v", err) @@ -302,7 +305,7 @@ func (c *ClusterController) setupStorageControllers(mgr manager.Manager, node *c LeafClient: mgr.GetClient(), RootClient: c.Root, RootClientSet: c.RootClient, - NodeName: node.Name, + NodeName: nodes[0].Name, } if err := leafPVController.SetupWithManager(mgr); err != nil { return fmt.Errorf("error starting leaf pv controller %v", err) @@ -310,35 +313,69 @@ func (c *ClusterController) setupStorageControllers(mgr manager.Manager, node *c return nil } -func (c *ClusterController) createNode(ctx context.Context, cluster *kosmosv1alpha1.Cluster, leafClient kubernetes.Interface) (*corev1.Node, error) { - nodeName := fmt.Sprintf("%s%s", utils.KosmosNodePrefix, cluster.Name) +func (c *ClusterController) createNode(ctx context.Context, cluster *kosmosv1alpha1.Cluster, leafClient kubernetes.Interface) ([]*corev1.Node, error) { + getNodeLen := func(cluster *kosmosv1alpha1.Cluster) int32 { + if cluster.Spec.ClusterTreeOptions.Enable { + return int32(len(cluster.Spec.ClusterTreeOptions.LeafModels)) + } + return 0 + } + serverVersion, err := leafClient.Discovery().ServerVersion() if err != nil { - klog.Errorf("create node failed, can not connect to leaf %s", nodeName) + klog.Errorf("create node failed, can not connect to leaf %s", cluster.Name) return nil, err } - node, err := c.RootClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) - if err != nil && !errors.IsNotFound(err) { - klog.Errorf("create node failed, can not get node %s", nodeName) - return nil, err + createOrUpdateNode := func(ctx context.Context, nodeName string) (*corev1.Node, error) { + node, err := c.RootClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil && !errors.IsNotFound(err) { + klog.Errorf("create node failed, can not get node %s", nodeName) + return nil, err + } + + if err != nil && errors.IsNotFound(err) { + node = utils.BuildNodeTemplate(nodeName) + node.Status.NodeInfo.KubeletVersion = serverVersion.GitVersion + node.Status.DaemonEndpoints = corev1.NodeDaemonEndpoints{ + KubeletEndpoint: corev1.DaemonEndpoint{ + Port: c.Options.ListenPort, + }, + } + node, err = c.RootClient.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + klog.Errorf("create node %s failed, err: %v", nodeName, err) + return nil, err + } + } + return node, nil } - if err != nil && errors.IsNotFound(err) { - node = utils.BuildNodeTemplate(nodeName) - node.Status.NodeInfo.KubeletVersion = serverVersion.GitVersion - node.Status.DaemonEndpoints = corev1.NodeDaemonEndpoints{ - KubeletEndpoint: corev1.DaemonEndpoint{ - Port: c.Options.ListenPort, - }, + nodes := make([]*corev1.Node, getNodeLen(cluster)) + + if getNodeLen(cluster) > 0 { + for _, leafModel := range cluster.Spec.ClusterTreeOptions.LeafModels { + // todo only support nodeName now + if leafModel.NodeSelector.NodeName != "" { + nodeName := leafModel.NodeSelector.NodeName + + node, err := createOrUpdateNode(ctx, nodeName) + if err != nil { + return nil, err + } + nodes = append(nodes, node) + } } - node, err = c.RootClient.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) - if err != nil && !errors.IsAlreadyExists(err) { - klog.Errorf("create node %s failed, err: %v", nodeName, err) + } else { + nodeName := fmt.Sprintf("%s%s", utils.KosmosNodePrefix, cluster.Name) + node, err := createOrUpdateNode(ctx, nodeName) + if err != nil { return nil, err } + nodes = append(nodes, node) } - return node, nil + + return nodes, nil } func (c *ClusterController) deleteNode(ctx context.Context, cluster *kosmosv1alpha1.Cluster) error { diff --git a/pkg/clustertree/cluster-manager/controllers/node_lease_controller.go b/pkg/clustertree/cluster-manager/controllers/node_lease_controller.go index ceda6c11a..ad7132eef 100644 --- a/pkg/clustertree/cluster-manager/controllers/node_lease_controller.go +++ b/pkg/clustertree/cluster-manager/controllers/node_lease_controller.go @@ -38,16 +38,16 @@ type NodeLeaseController struct { leaseInterval time.Duration statusInterval time.Duration - node *corev1.Node + nodes []*corev1.Node nodeLock sync.Mutex } -func NewNodeLeaseController(leafClient kubernetes.Interface, root client.Client, node *corev1.Node, rootClient kubernetes.Interface) *NodeLeaseController { +func NewNodeLeaseController(leafClient kubernetes.Interface, root client.Client, nodes []*corev1.Node, rootClient kubernetes.Interface) *NodeLeaseController { c := &NodeLeaseController{ leafClient: leafClient, rootClient: rootClient, root: root, - node: node, + nodes: nodes, leaseInterval: getRenewInterval(), statusInterval: DefaultNodeStatusUpdateInterval, } @@ -64,7 +64,7 @@ func (c *NodeLeaseController) Start(ctx context.Context) error { func (c *NodeLeaseController) syncNodeStatus(ctx context.Context) { c.nodeLock.Lock() - node := c.node.DeepCopy() + node := c.nodes[0].DeepCopy() c.nodeLock.Unlock() err := c.updateNodeStatus(ctx, node) @@ -103,12 +103,12 @@ func (c *NodeLeaseController) updateNodeStatus(ctx context.Context, n *corev1.No func (c *NodeLeaseController) syncLease(ctx context.Context) { c.nodeLock.Lock() - node := c.node.DeepCopy() + node := c.nodes[0].DeepCopy() c.nodeLock.Unlock() _, err := c.leafClient.Discovery().ServerVersion() if err != nil { - klog.Errorf("failed to ping leaf %s", c.node.Name) + klog.Errorf("failed to ping leaf %s", c.nodes[0].Name) return } diff --git a/pkg/clustertree/cluster-manager/controllers/node_resources_controller.go b/pkg/clustertree/cluster-manager/controllers/node_resources_controller.go index cf9c14e5c..84cd14226 100644 --- a/pkg/clustertree/cluster-manager/controllers/node_resources_controller.go +++ b/pkg/clustertree/cluster-manager/controllers/node_resources_controller.go @@ -22,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + kosmosv1alpha1 "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/utils" ) @@ -35,7 +36,8 @@ type NodeResourcesController struct { Root client.Client RootClientset kubernetes.Interface - Node *corev1.Node + Nodes []*corev1.Node + Cluster *kosmosv1alpha1.Cluster EventRecorder record.EventRecorder } @@ -106,11 +108,11 @@ func (c *NodeResourcesController) Reconcile(ctx context.Context, request reconci clusterResources := utils.CalculateClusterResources(&nodes, &pods) node := &corev1.Node{} - err := c.Root.Get(ctx, types.NamespacedName{Name: c.Node.Name}, node) + err := c.Root.Get(ctx, types.NamespacedName{Name: c.Nodes[0].Name}, node) if err != nil { return reconcile.Result{ RequeueAfter: RequeueTime, - }, fmt.Errorf("cannot get node while update node resources %s, err: %v", c.Node.Name, err) + }, fmt.Errorf("cannot get node while update node resources %s, err: %v", c.Nodes[0].Name, err) } clone := node.DeepCopy() @@ -123,7 +125,7 @@ func (c *NodeResourcesController) Reconcile(ctx context.Context, request reconci return reconcile.Result{}, err } - if _, err := c.RootClientset.CoreV1().Nodes().PatchStatus(ctx, c.Node.Name, patch); err != nil { + if _, err := c.RootClientset.CoreV1().Nodes().PatchStatus(ctx, c.Nodes[0].Name, patch); err != nil { return reconcile.Result{ RequeueAfter: RequeueTime, }, fmt.Errorf("failed to patch node resources: %v, will requeue", err) diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 5a1f9d783..76404964d 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -636,7 +636,7 @@ func schema_pkg_apis_kosmos_v1alpha1_ClusterTreeOptions(ref common.ReferenceCall }, "leafModel": { SchemaProps: spec.SchemaProps{ - Description: "LeafModel provide an api to arrange the member cluster with some rules to pretend one or more leaf node If nil or empty, the hole member cluster nodes will pretend one leaf node.", + Description: "LeafModels provide an api to arrange the member cluster with some rules to pretend one or more leaf node", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{