From 000fb3575a1baf61edb72d7404f9e2933b88ba10 Mon Sep 17 00:00:00 2001 From: wangyizhi1 Date: Tue, 21 Nov 2023 18:21:05 +0800 Subject: [PATCH 1/2] fix: use root node label first Signed-off-by: wangyizhi1 --- .../cluster-manager/controllers/node_resources_controller.go | 4 ++-- pkg/clustertree/cluster-manager/utils/leaf_model_handler.go | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/clustertree/cluster-manager/controllers/node_resources_controller.go b/pkg/clustertree/cluster-manager/controllers/node_resources_controller.go index 34cb67a96..d4982b28d 100644 --- a/pkg/clustertree/cluster-manager/controllers/node_resources_controller.go +++ b/pkg/clustertree/cluster-manager/controllers/node_resources_controller.go @@ -144,8 +144,8 @@ func (c *NodeResourcesController) Reconcile(ctx context.Context, request reconci } node := getNode(nodesInLeaf) if node != nil { - clone.Labels = mergeMap(clone.GetLabels(), node.GetLabels()) - clone.Annotations = mergeMap(clone.GetAnnotations(), node.GetAnnotations()) + clone.Labels = mergeMap(node.GetLabels(), clone.GetLabels()) + clone.Annotations = mergeMap(node.GetAnnotations(), clone.GetAnnotations()) spec := corev1.NodeSpec{ Taints: rootNode.Spec.Taints, } diff --git a/pkg/clustertree/cluster-manager/utils/leaf_model_handler.go b/pkg/clustertree/cluster-manager/utils/leaf_model_handler.go index 8938ada02..1bbf092e1 100644 --- a/pkg/clustertree/cluster-manager/utils/leaf_model_handler.go +++ b/pkg/clustertree/cluster-manager/utils/leaf_model_handler.go @@ -212,8 +212,11 @@ func (h DispersionModelHandler) UpdateNodeStatus(ctx context.Context, n []*corev return fmt.Errorf("cannot get node in root cluster while update node status %s, err: %v", nodeCopy.Name, err) } + rootCopy := nodeRoot.DeepCopy() nodeRoot.Status = nodeInLeaf.Status nodeRoot.Status.Addresses = GetAddress() + nodeRoot.Status.Allocatable = rootCopy.Status.Allocatable + nodeRoot.Status.Capacity = rootCopy.Status.Capacity if node, err = h.RootClientset.CoreV1().Nodes().UpdateStatus(ctx, nodeRoot, metav1.UpdateOptions{}); err != nil { return err From 58b05eca6b9c82d50ca5a3be9998b9a74bb4af47 Mon Sep 17 00:00:00 2001 From: wangyizhi1 Date: Tue, 21 Nov 2023 16:09:24 +0800 Subject: [PATCH 2/2] fix: incorrect node resources Signed-off-by: wangyizhi1 --- .../controllers/node_resources_controller.go | 7 +++--- .../controllers/pv/oneway_pv_controller.go | 4 ++-- .../controllers/pvc/oneway_pvc_controller.go | 4 ++-- pkg/utils/resources.go | 22 +++++++++++++++++++ 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/pkg/clustertree/cluster-manager/controllers/node_resources_controller.go b/pkg/clustertree/cluster-manager/controllers/node_resources_controller.go index d4982b28d..ec3aee9dc 100644 --- a/pkg/clustertree/cluster-manager/controllers/node_resources_controller.go +++ b/pkg/clustertree/cluster-manager/controllers/node_resources_controller.go @@ -126,10 +126,7 @@ func (c *NodeResourcesController) Reconcile(ctx context.Context, request reconci }, err } - clusterResources := utils.CalculateClusterResources(nodesInLeaf, pods) clone := nodeInRoot.DeepCopy() - clone.Status.Allocatable = clusterResources - clone.Status.Capacity = clusterResources clone.Status.Conditions = utils.NodeConditions() // Node2Node mode should sync leaf node's labels and annotations to root nodeInRoot @@ -156,6 +153,10 @@ func (c *NodeResourcesController) Reconcile(ctx context.Context, request reconci } } + clusterResources := utils.CalculateClusterResources(nodesInLeaf, pods) + clone.Status.Allocatable = clusterResources + clone.Status.Capacity = clusterResources + patch, err := utils.CreateMergePatch(nodeInRoot, clone) if err != nil { klog.Errorf("Could not CreateMergePatch,Error: %v", err) diff --git a/pkg/clustertree/cluster-manager/controllers/pv/oneway_pv_controller.go b/pkg/clustertree/cluster-manager/controllers/pv/oneway_pv_controller.go index 7b0967cb9..65e6ff62d 100644 --- a/pkg/clustertree/cluster-manager/controllers/pv/oneway_pv_controller.go +++ b/pkg/clustertree/cluster-manager/controllers/pv/oneway_pv_controller.go @@ -87,8 +87,8 @@ func (c *OnewayPVController) Reconcile(ctx context.Context, request reconcile.Re vp, err := c.RootDynamic.Resource(VolumePathGVR).Get(ctx, request.Name, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { - klog.Warningf("vp %s not found", request.Name) - return reconcile.Result{}, nil + klog.V(4).Infof("vp %s not found", request.Name) + return reconcile.Result{RequeueAfter: requeueTime}, nil } klog.Errorf("get volumePath %s error: %v", request.Name, err) return reconcile.Result{RequeueAfter: requeueTime}, nil diff --git a/pkg/clustertree/cluster-manager/controllers/pvc/oneway_pvc_controller.go b/pkg/clustertree/cluster-manager/controllers/pvc/oneway_pvc_controller.go index fe8e7d654..c6f005a46 100644 --- a/pkg/clustertree/cluster-manager/controllers/pvc/oneway_pvc_controller.go +++ b/pkg/clustertree/cluster-manager/controllers/pvc/oneway_pvc_controller.go @@ -96,8 +96,8 @@ func (c *OnewayPVCController) Reconcile(ctx context.Context, request reconcile.R vp, err := c.RootDynamic.Resource(VolumePathGVR).Get(ctx, request.Name, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { - klog.Warningf("vp %s not found", request.Name) - return reconcile.Result{}, nil + klog.V(4).Infof("vp %s not found", request.Name) + return reconcile.Result{RequeueAfter: requeueTime}, nil } klog.Errorf("get volumePath %s error: %v", request.Name, err) return reconcile.Result{RequeueAfter: requeueTime}, nil diff --git a/pkg/utils/resources.go b/pkg/utils/resources.go index 161f1618d..13fdee5df 100644 --- a/pkg/utils/resources.go +++ b/pkg/utils/resources.go @@ -2,13 +2,20 @@ package utils import ( corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" v1resource "k8s.io/kubernetes/pkg/api/v1/resource" ) +const ( + podResourceName corev1.ResourceName = "pods" +) + func CalculateClusterResources(nodes *corev1.NodeList, pods *corev1.PodList) corev1.ResourceList { base := GetNodesTotalResources(nodes) reqs, _ := GetPodsTotalRequestsAndLimits(pods) + podNums := GetUsedPodNums(pods) SubResourceList(base, reqs) + SubResourceList(base, podNums) return base } @@ -70,3 +77,18 @@ func GetPodsTotalRequestsAndLimits(podList *corev1.PodList) (reqs corev1.Resourc } return } + +func GetUsedPodNums(podList *corev1.PodList) (res corev1.ResourceList) { + podQuantity := resource.Quantity{} + res = corev1.ResourceList{} + for _, p := range podList.Items { + pod := p + if IsVirtualPod(&pod) { + continue + } + q := resource.MustParse("1") + podQuantity.Add(q) + } + res[podResourceName] = podQuantity + return +}