From 53036b316ed309cebb971b025fca6086fdf72abd Mon Sep 17 00:00:00 2001 From: ONE7live Date: Wed, 22 Nov 2023 17:04:34 +0800 Subject: [PATCH 1/8] fix: unjoin exception and init leaf model Signed-off-by: ONE7live --- pkg/kosmosctl/install/install.go | 24 ---- pkg/kosmosctl/join/join.go | 75 +++++++++- pkg/kosmosctl/manifest/manifest_crds.go | 112 ++++++++------- pkg/kosmosctl/uninstall/uninstall.go | 2 +- pkg/kosmosctl/unjoin/unjoin.go | 183 ++++++++++++++---------- pkg/kosmosctl/util/builder.go | 1 - pkg/utils/constants.go | 2 + 7 files changed, 236 insertions(+), 163 deletions(-) diff --git a/pkg/kosmosctl/install/install.go b/pkg/kosmosctl/install/install.go index c25fdd4eb..12901b3c7 100644 --- a/pkg/kosmosctl/install/install.go +++ b/pkg/kosmosctl/install/install.go @@ -404,30 +404,6 @@ func (o *CommandInstallOptions) runClustertree() error { } klog.Info("Create CRD " + clustertreeCluster.Name + " successful.") - serviceExport, err := util.GenerateCustomResourceDefinition(manifest.ServiceExport, nil) - if err != nil { - return err - } - _, err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), serviceExport, metav1.CreateOptions{}) - if err != nil { - if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl install clustertree run error, crd options failed: %v", err) - } - } - klog.Info("Create CRD " + serviceExport.Name + " successful.") - - serviceImport, err := util.GenerateCustomResourceDefinition(manifest.ServiceImport, nil) - if err != nil { - return err - } - _, err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), serviceImport, metav1.CreateOptions{}) - if err != nil { - if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl install clustertree run error, crd options failed: %v", err) - } - } - klog.Info("Create CRD " + serviceImport.Name + " successful.") - klog.Info("Start creating kosmos-clustertree ConfigMap...") clustertreeConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/kosmosctl/join/join.go b/pkg/kosmosctl/join/join.go index a307adc6f..d09838dee 100644 --- a/pkg/kosmosctl/join/join.go +++ b/pkg/kosmosctl/join/join.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" + extensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -60,9 +61,11 @@ type CommandJoinOptions struct { UseProxy string EnableTree bool + LeafModel string - KosmosClient versioned.Interface - K8sClient kubernetes.Interface + KosmosClient versioned.Interface + K8sClient kubernetes.Interface + K8sExtensionsClient extensionsclient.Interface } // NewCmdJoin join resource to Kosmos control plane. @@ -99,6 +102,7 @@ func NewCmdJoin(f ctlutil.Factory) *cobra.Command { flags.StringVar(&o.IpFamily, "ip-family", utils.DefaultIPv4, "Specify the IP protocol version used by network devices, common IP families include IPv4 and IPv6.") flags.StringVar(&o.UseProxy, "use-proxy", "false", "Set whether to enable proxy.") flags.BoolVar(&o.EnableTree, "enable-tree", false, "Turn on clustertree.") + flags.StringVar(&o.LeafModel, "leaf-model", "", "Set leaf cluster model, which supports one-to-one model.") flags.IntVarP(&o.WaitTime, "wait-time", "", utils.DefaultWaitTime, "Wait the specified time for the Kosmos install ready.") return cmd @@ -147,7 +151,12 @@ func (o *CommandJoinOptions) Complete(f ctlutil.Factory) error { o.K8sClient, err = kubernetes.NewForConfig(clusterConfig) if err != nil { - return fmt.Errorf("kosmosctl join complete error, generate basic client failed: %v", err) + return fmt.Errorf("kosmosctl join complete error, generate K8s basic client failed: %v", err) + } + + o.K8sExtensionsClient, err = extensionsclient.NewForConfig(clusterConfig) + if err != nil { + return fmt.Errorf("kosmosctl join complete error, generate K8s extensions client failed: %v", err) } } else { return fmt.Errorf("kosmosctl join complete error, arg ClusterKubeConfig is required") @@ -246,10 +255,62 @@ func (o *CommandJoinOptions) runCluster() error { cluster.Spec.ClusterLinkOptions.CNI = o.CNI } - // ToDo ClusterTree currently has no init parameters, can be expanded later. - //if o.EnableTree { - // - //} + if o.EnableTree { + serviceExport, err := util.GenerateCustomResourceDefinition(manifest.ServiceExport, nil) + if err != nil { + return err + } + _, err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), serviceExport, metav1.CreateOptions{}) + if err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("kosmosctl join run error, crd options failed: %v", err) + } + } + klog.Info("Create CRD " + serviceExport.Name + " successful.") + + serviceImport, err := util.GenerateCustomResourceDefinition(manifest.ServiceImport, nil) + if err != nil { + return err + } + _, err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), serviceImport, metav1.CreateOptions{}) + if err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("kosmosctl join run error, crd options failed: %v", err) + } + } + klog.Info("Create CRD " + serviceImport.Name + " successful.") + + if len(o.LeafModel) > 0 { + switch o.LeafModel { + case "one-to-one": + // ToDo Perform follow-up query based on the leaf cluster label + nodes, err := o.K8sClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{ + LabelSelector: utils.KosmosNodeJoinLabel + "=" + utils.KosmosNodeJoinValue, + }) + if err != nil { + return fmt.Errorf("kosmosctl join run error, list cluster node failed: %v", err) + } + var leafModels []v1alpha1.LeafModel + for _, n := range nodes.Items { + leafModel := v1alpha1.LeafModel{ + LeafNodeName: n.Name, + Taints: []corev1.Taint{ + { + Effect: utils.KosmosNodeTaintEffect, + Key: utils.KosmosNodeTaintKey, + Value: utils.KosmosNodeValue, + }, + }, + NodeSelector: v1alpha1.NodeSelector{ + NodeName: n.Name, + }, + } + leafModels = append(leafModels, leafModel) + } + cluster.Spec.ClusterTreeOptions.LeafModels = leafModels + } + } + } if o.RootFlag { cluster.Annotations = map[string]string{ diff --git a/pkg/kosmosctl/manifest/manifest_crds.go b/pkg/kosmosctl/manifest/manifest_crds.go index 20314e586..1174c1550 100644 --- a/pkg/kosmosctl/manifest/manifest_crds.go +++ b/pkg/kosmosctl/manifest/manifest_crds.go @@ -491,60 +491,11 @@ spec: enable: default: true type: boolean - leafModel: - description: LeafModel provide an api to arrange the member cluster + leafModels: + description: LeafModels provide an api to arrange the member cluster with some rules to pretend one or more leaf node items: properties: - labelSelector: - description: LabelSelector is a filter to select member - cluster nodes to pretend a leaf node in clusterTree by - labels. If nil or empty, the hole member cluster nodes - will pretend one leaf node. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector - that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, - NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. - If the operator is In or NotIn, the values array - must be non-empty. If the operator is Exists - or DoesNotExist, the values array must be empty. - This array is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. - A single {key,value} in the matchLabels map is equivalent - to an element of matchExpressions, whose key field - is "key", the operator is "In", and the values array - contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic labels: additionalProperties: type: string @@ -556,6 +507,65 @@ spec: the leaf node name will generate by controller and fill in cluster link status type: string + nodeSelector: + description: NodeSelector is a selector to select member + cluster nodes to pretend a leaf node in clusterTree. + properties: + labelSelector: + description: LabelSelector is a filter to select member + cluster nodes to pretend a leaf node in clusterTree + by labels. It will work on second level schedule on + pod create in member clusters. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + nodeName: + description: NodeName is Member cluster origin node + Name + type: string + type: object taints: description: Taints attached to the leaf pretended Node. If nil or empty, controller will set the default no-schedule diff --git a/pkg/kosmosctl/uninstall/uninstall.go b/pkg/kosmosctl/uninstall/uninstall.go index 920147052..97930452b 100644 --- a/pkg/kosmosctl/uninstall/uninstall.go +++ b/pkg/kosmosctl/uninstall/uninstall.go @@ -319,7 +319,7 @@ func (o *CommandUninstallOptions) runClustertree() error { } } else { klog.Info("Deployment " + clustertreeDeploy.Name + " is deleted.") - clustertreeSecret, err := util.GenerateService(manifest.ClusterTreeClusterManagerSecret, manifest.SecretReplace{ + clustertreeSecret, err := util.GenerateSecret(manifest.ClusterTreeClusterManagerSecret, manifest.SecretReplace{ Namespace: o.Namespace, Cert: "", Key: "", diff --git a/pkg/kosmosctl/unjoin/unjoin.go b/pkg/kosmosctl/unjoin/unjoin.go index 82973db6c..24ed7fcfc 100644 --- a/pkg/kosmosctl/unjoin/unjoin.go +++ b/pkg/kosmosctl/unjoin/unjoin.go @@ -6,9 +6,9 @@ import ( "time" "github.com/spf13/cobra" + extensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -17,24 +17,29 @@ import ( "k8s.io/kubectl/pkg/util/i18n" "k8s.io/kubectl/pkg/util/templates" + "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" + "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" "github.com/kosmos.io/kosmos/pkg/kosmosctl/manifest" "github.com/kosmos.io/kosmos/pkg/kosmosctl/util" "github.com/kosmos.io/kosmos/pkg/utils" ) var unjoinExample = templates.Examples(i18n.T(` - # Unjoin cluster from Kosmos control plane, e.g: - kosmosctl unjoin cluster --name cluster-name --kubeconfig ~/kubeconfig/cluster-kubeconfig -`)) + # Unjoin cluster from Kosmos control plane, e.g: + kosmosctl unjoin cluster --name cluster-name + + # Unjoin cluster from Kosmos control plane, if you need to specify a special cluster kubeconfig, e.g: + kosmosctl unjoin cluster --name cluster-name --kubeconfig ~/kubeconfig/cluster-kubeconfig`)) type CommandUnJoinOptions struct { + Name string + Namespace string KubeConfig string HostKubeConfig string - Name string - - Client kubernetes.Interface - DynamicClient *dynamic.DynamicClient + KosmosClient versioned.Interface + K8sClient kubernetes.Interface + K8sExtensionsClient extensionsclient.Interface } // NewCmdUnJoin Delete resource in Kosmos control plane. @@ -43,7 +48,7 @@ func NewCmdUnJoin(f ctlutil.Factory) *cobra.Command { cmd := &cobra.Command{ Use: "unjoin", - Short: i18n.T("Unjoin resource in kosmos control plane"), + Short: i18n.T("Unjoin resource from Kosmos control plane"), Long: "", Example: unjoinExample, SilenceUsage: true, @@ -56,14 +61,16 @@ func NewCmdUnJoin(f ctlutil.Factory) *cobra.Command { }, } + cmd.Flags().StringVar(&o.Name, "name", "", "Specify the name of the resource to unjoin.") + cmd.Flags().StringVarP(&o.Namespace, "namespace", "n", utils.DefaultNamespace, "Kosmos namespace.") cmd.Flags().StringVar(&o.KubeConfig, "kubeconfig", "", "Absolute path to the cluster kubeconfig file.") cmd.Flags().StringVar(&o.HostKubeConfig, "host-kubeconfig", "", "Absolute path to the special host kubeconfig file.") - cmd.Flags().StringVar(&o.Name, "name", "", "Specify the name of the resource to unjoin.") return cmd } func (o *CommandUnJoinOptions) Complete(f ctlutil.Factory) error { var hostConfig *restclient.Config + var clusterConfig *restclient.Config var err error if o.HostKubeConfig != "" { @@ -78,19 +85,36 @@ func (o *CommandUnJoinOptions) Complete(f ctlutil.Factory) error { } } - clusterConfig, err := clientcmd.BuildConfigFromFlags("", o.KubeConfig) + o.KosmosClient, err = versioned.NewForConfig(hostConfig) if err != nil { - return fmt.Errorf("kosmosctl unjoin complete error, generate clusterConfig failed: %s", err) + return fmt.Errorf("kosmosctl install complete error, generate Kosmos client failed: %v", err) } - o.Client, err = kubernetes.NewForConfig(clusterConfig) + if o.KubeConfig != "" { + clusterConfig, err = clientcmd.BuildConfigFromFlags("", o.KubeConfig) + if err != nil { + return fmt.Errorf("kosmosctl unjoin complete error, generate clusterConfig failed: %s", err) + } + } else { + var cluster *v1alpha1.Cluster + cluster, err = o.KosmosClient.KosmosV1alpha1().Clusters().Get(context.TODO(), o.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("kosmosctl unjoin complete error, get cluster failed: %s", err) + } + clusterConfig, err = clientcmd.RESTConfigFromKubeConfig(cluster.Spec.Kubeconfig) + if err != nil { + return fmt.Errorf("kosmosctl unjoin complete error, generate clusterConfig failed: %s", err) + } + } + + o.K8sClient, err = kubernetes.NewForConfig(clusterConfig) if err != nil { - return fmt.Errorf("kosmosctl join complete error, generate basic client failed: %v", err) + return fmt.Errorf("kosmosctl unjoin complete error, generate K8s basic client failed: %v", err) } - o.DynamicClient, err = dynamic.NewForConfig(hostConfig) + o.K8sExtensionsClient, err = extensionsclient.NewForConfig(clusterConfig) if err != nil { - return fmt.Errorf("kosmosctl unjoin complete error, generate dynamic client failed: %s", err) + return fmt.Errorf("kosmosctl unjoin complete error, generate K8s extensions client failed: %v", err) } return nil @@ -101,25 +125,6 @@ func (o *CommandUnJoinOptions) Validate(args []string) error { return fmt.Errorf("kosmosctl unjoin validate error, name is not valid") } - switch args[0] { - case "cluster": - _, err := o.DynamicClient.Resource(util.ClusterGVR).Get(context.TODO(), o.Name, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return fmt.Errorf("kosmosctl unjoin validate warning, clsuter is not found: %s", err) - } - return fmt.Errorf("kosmosctl unjoin validate error, get cluster failed: %s", err) - } - case "knode": - _, err := o.DynamicClient.Resource(util.KnodeGVR).Get(context.TODO(), o.Name, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return fmt.Errorf("kosmosctl unjoin validate warning, knode is not found: %s", err) - } - return fmt.Errorf("kosmosctl unjoin validate error, get knode failed: %s", err) - } - } - return nil } @@ -130,11 +135,6 @@ func (o *CommandUnJoinOptions) Run(args []string) error { if err != nil { return err } - case "knode": - err := o.runKnode() - if err != nil { - return err - } } return nil @@ -142,9 +142,9 @@ func (o *CommandUnJoinOptions) Run(args []string) error { func (o *CommandUnJoinOptions) runCluster() error { klog.Info("Start removing cluster from kosmos control plane...") - // 1. delete cluster + // delete cluster for { - err := o.DynamicClient.Resource(util.ClusterGVR).Namespace("").Delete(context.TODO(), o.Name, metav1.DeleteOptions{}) + err := o.KosmosClient.KosmosV1alpha1().Clusters().Delete(context.TODO(), o.Name, metav1.DeleteOptions{}) if err != nil { if apierrors.IsNotFound(err) { break @@ -155,55 +155,97 @@ func (o *CommandUnJoinOptions) runCluster() error { } klog.Info("Cluster: " + o.Name + " has been deleted.") - // 2. delete operator - clusterlinkOperatorDeployment, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, nil) + // delete crd + serviceExport, err := util.GenerateCustomResourceDefinition(manifest.ServiceExport, nil) if err != nil { return err } - err = o.Client.AppsV1().Deployments(utils.DefaultNamespace).Delete(context.TODO(), clusterlinkOperatorDeployment.Name, metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("kosmosctl unjoin run error, delete deployment failed: %s", err) + err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.Background(), serviceExport.Name, metav1.DeleteOptions{}) + if err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("kosmosctl unjoin run error, crd options failed: %v", err) + } } - klog.Info("Deployment: " + clusterlinkOperatorDeployment.Name + " has been deleted.") + klog.Info("CRD: " + serviceExport.Name + " has been deleted.") - // 3. delete secret - err = o.Client.CoreV1().Secrets(utils.DefaultNamespace).Delete(context.TODO(), utils.ControlPanelSecretName, metav1.DeleteOptions{}) + serviceImport, err := util.GenerateCustomResourceDefinition(manifest.ServiceImport, nil) + if err != nil { + return err + } + err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.Background(), serviceImport.Name, metav1.DeleteOptions{}) + if err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("kosmosctl unjoin run error, crd options failed: %v", err) + } + } + klog.Info("CRD: " + serviceImport.Name + " has been deleted.") + + // delete rbac + err = o.K8sClient.CoreV1().Secrets(o.Namespace).Delete(context.TODO(), utils.ControlPanelSecretName, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("kosmosctl unjoin run error, delete secret failed: %s", err) } klog.Info("Secret: " + utils.ControlPanelSecretName + " has been deleted.") - // 4. delete rbac - err = o.Client.RbacV1().ClusterRoleBindings().Delete(context.TODO(), utils.ExternalIPPoolNamePrefix, metav1.DeleteOptions{}) + err = o.K8sClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), utils.ExternalIPPoolNamePrefix, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("kosmosctl unjoin run error, delete clusterrolebinding failed: %s", err) } klog.Info("ClusterRoleBinding: " + utils.ExternalIPPoolNamePrefix + " has been deleted.") - err = o.Client.RbacV1().ClusterRoles().Delete(context.TODO(), utils.ExternalIPPoolNamePrefix, metav1.DeleteOptions{}) + err = o.K8sClient.RbacV1().ClusterRoles().Delete(context.TODO(), utils.ExternalIPPoolNamePrefix, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("kosmosctl unjoin run error, delete clusterrole failed: %s", err) } klog.Info("ClusterRole: " + utils.ExternalIPPoolNamePrefix + " has been deleted.") - clusterlinkOperatorServiceAccount, err := util.GenerateServiceAccount(manifest.KosmosOperatorServiceAccount, nil) + kosmosCR, err := util.GenerateClusterRole(manifest.KosmosClusterRole, nil) + if err != nil { + return fmt.Errorf("kosmosctl unjoin run error, generate clusterrole failed: %s", err) + } + err = o.K8sClient.RbacV1().ClusterRoles().Delete(context.TODO(), kosmosCR.Name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("kosmosctl unjoin run error, delete clusterrole failed: %s", err) + } + klog.Info("ClusterRole " + kosmosCR.Name + " has been deleted.") + + kosmosCRB, err := util.GenerateClusterRoleBinding(manifest.KosmosClusterRoleBinding, manifest.ClusterRoleBindingReplace{ + Namespace: o.Namespace, + }) + if err != nil { + return fmt.Errorf("kosmosctl join run error, generate clusterrolebinding failed: %s", err) + } + err = o.K8sClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), kosmosCRB.Name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("kosmosctl unjoin run error, delete clusterrolebinding failed: %s", err) + } + klog.Info("ClusterRoleBinding " + kosmosCRB.Name + " has been deleted.") + + kosmosOperatorSA, err := util.GenerateServiceAccount(manifest.KosmosOperatorServiceAccount, nil) if err != nil { return err } - err = o.Client.CoreV1().ServiceAccounts(utils.DefaultNamespace).Delete(context.TODO(), clusterlinkOperatorServiceAccount.Name, metav1.DeleteOptions{}) + err = o.K8sClient.CoreV1().ServiceAccounts(o.Namespace).Delete(context.TODO(), kosmosOperatorSA.Name, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("kosmosctl unjoin run error, delete serviceaccout failed: %s", err) } - klog.Info("ServiceAccount: " + clusterlinkOperatorServiceAccount.Name + " has been deleted.") + klog.Info("ServiceAccount: " + kosmosOperatorSA.Name + " has been deleted.") - // 5. If cluster is not the master, delete namespace - clusterlinkNetworkManagerDeployment, err := util.GenerateDeployment(manifest.ClusterlinkNetworkManagerDeployment, nil) + kosmosControlSA, err := util.GenerateServiceAccount(manifest.KosmosControlServiceAccount, manifest.ServiceAccountReplace{ + Namespace: o.Namespace, + }) if err != nil { - return err + return fmt.Errorf("kosmosctl unjoin run error, generate serviceaccount failed: %s", err) + } + err = o.K8sClient.CoreV1().ServiceAccounts(kosmosControlSA.Namespace).Delete(context.TODO(), kosmosControlSA.Name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("kosmosctl unjoin run error, delete serviceaccount failed: %s", err) } - _, err = o.Client.AppsV1().Deployments(utils.DefaultNamespace).Get(context.TODO(), clusterlinkNetworkManagerDeployment.Name, metav1.GetOptions{}) - if err != nil && apierrors.IsNotFound(err) { - err = o.Client.CoreV1().Namespaces().Delete(context.TODO(), utils.DefaultNamespace, metav1.DeleteOptions{}) + klog.Info("ServiceAccount " + kosmosControlSA.Name + " has been deleted.") + + // if cluster is not the master, delete namespace + if o.Name != utils.DefaultClusterName { + err = o.K8sClient.CoreV1().Namespaces().Delete(context.TODO(), o.Namespace, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("kosmosctl unjoin run error, delete namespace failed: %s", err) } @@ -212,20 +254,3 @@ func (o *CommandUnJoinOptions) runCluster() error { klog.Info("Cluster [" + o.Name + "] is removed.") return nil } - -func (o *CommandUnJoinOptions) runKnode() error { - klog.Info("Start removing knode from kosmos control plane...") - for { - err := o.DynamicClient.Resource(util.KnodeGVR).Namespace("").Delete(context.TODO(), o.Name, metav1.DeleteOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - break - } - return fmt.Errorf("kosmosctl unjoin run error, delete knode failed: %s", err) - } - time.Sleep(3 * time.Second) - } - - klog.Info("Knode [" + o.Name + "] is removed.") - return nil -} diff --git a/pkg/kosmosctl/util/builder.go b/pkg/kosmosctl/util/builder.go index b29aa4d73..db8c20fcf 100644 --- a/pkg/kosmosctl/util/builder.go +++ b/pkg/kosmosctl/util/builder.go @@ -18,7 +18,6 @@ var ( ClusterGVR = schema.GroupVersionResource{Group: "kosmos.io", Version: "v1alpha1", Resource: "clusters"} ClusterNodeGVR = schema.GroupVersionResource{Group: "kosmos.io", Version: "v1alpha1", Resource: "clusternodes"} NodeConfigGVR = schema.GroupVersionResource{Group: "kosmos.io", Version: "v1alpha1", Resource: "nodeconfigs"} - KnodeGVR = schema.GroupVersionResource{Group: "kosmos.io", Version: "v1alpha1", Resource: "knodes"} ) func GenerateDeployment(deployTemplate string, obj interface{}) (*appsv1.Deployment, error) { diff --git a/pkg/utils/constants.go b/pkg/utils/constants.go index 37ff95e98..6b88f8539 100644 --- a/pkg/utils/constants.go +++ b/pkg/utils/constants.go @@ -66,6 +66,8 @@ const ( KosmosNodePrefix = "kosmos-" KosmosNodeLabel = "kosmos.io/node" KosmosNodeValue = "true" + KosmosNodeJoinLabel = "kosmos.io/join" + KosmosNodeJoinValue = "true" KosmosNodeTaintKey = "kosmos.io/node" KosmosNodeTaintValue = "true" KosmosNodeTaintEffect = "NoSchedule" From cbf10ac39b9a005b12cc77df65d96de8259770d8 Mon Sep 17 00:00:00 2001 From: renxiangyu Date: Wed, 22 Nov 2023 18:02:40 +0800 Subject: [PATCH 2/8] fix: available pod is negative Signed-off-by: renxiangyu --- pkg/utils/resources.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/utils/resources.go b/pkg/utils/resources.go index 13fdee5df..74ba65e29 100644 --- a/pkg/utils/resources.go +++ b/pkg/utils/resources.go @@ -13,7 +13,7 @@ const ( func CalculateClusterResources(nodes *corev1.NodeList, pods *corev1.PodList) corev1.ResourceList { base := GetNodesTotalResources(nodes) reqs, _ := GetPodsTotalRequestsAndLimits(pods) - podNums := GetUsedPodNums(pods) + podNums := GetUsedPodNums(pods, nodes) SubResourceList(base, reqs) SubResourceList(base, podNums) return base @@ -78,14 +78,22 @@ func GetPodsTotalRequestsAndLimits(podList *corev1.PodList) (reqs corev1.Resourc return } -func GetUsedPodNums(podList *corev1.PodList) (res corev1.ResourceList) { +func GetUsedPodNums(podList *corev1.PodList, nodes *corev1.NodeList) (res corev1.ResourceList) { podQuantity := resource.Quantity{} res = corev1.ResourceList{} + nodeMap := map[string]corev1.Node{} + for _, item := range nodes.Items { + nodeMap[item.Name] = item + } for _, p := range podList.Items { pod := p if IsVirtualPod(&pod) { continue } + node, exists := nodeMap[pod.Spec.NodeName] + if !exists || node.Spec.Unschedulable || !NodeReady(&node) { + continue + } q := resource.MustParse("1") podQuantity.Add(q) } From 749c2f738309b46b5243c12f8d8520a3c4656b76 Mon Sep 17 00:00:00 2001 From: wuyingjun Date: Thu, 23 Nov 2023 10:41:58 +0800 Subject: [PATCH 3/8] modify flannel config informer options to avoid the situation when the namespace is custom Signed-off-by: wuyingjun --- pkg/clusterlink/controllers/cluster/cluster_controller.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/clusterlink/controllers/cluster/cluster_controller.go b/pkg/clusterlink/controllers/cluster/cluster_controller.go index 89eed5c1e..a99fdcde7 100644 --- a/pkg/clusterlink/controllers/cluster/cluster_controller.go +++ b/pkg/clusterlink/controllers/cluster/cluster_controller.go @@ -39,7 +39,6 @@ import ( // KubeFlannelNetworkConfig const ( FlannelCNI = "flannel" - KubeFlannelNamespace = "kube-flannel" KubeFlannelConfigMap = "kube-flannel-cfg" KubeFlannelNetworkConf = "net-conf.json" KubeFlannelIPPool = "Network" @@ -371,7 +370,7 @@ func (c *Controller) initCalicoWatcherWithEtcdBackend(ctx context.Context, clust // todo by wuyingjun-lucky func (c *Controller) initFlannelInformer(context context.Context, cluster *clusterlinkv1alpha1.Cluster, kubeClient kubernetes.Interface) (SetClusterPodCIDRFun, error) { - informerFactory := informers.NewSharedInformerFactoryWithOptions(kubeClient, 0, informers.WithNamespace(KubeFlannelNamespace)) + informerFactory := informers.NewSharedInformerFactory(kubeClient, 0) lister := informerFactory.Core().V1().ConfigMaps().Lister() _, err := informerFactory.Core().V1().ConfigMaps().Informer().AddEventHandler( cache.ResourceEventHandlerFuncs{ From ae4f0b4d053d82c0a275e2239b911a98f11eae87 Mon Sep 17 00:00:00 2001 From: wuyingjun Date: Thu, 23 Nov 2023 21:06:54 +0800 Subject: [PATCH 4/8] add clusterlink code owners Signed-off-by: wuyingjun --- cmd/clusterlink/OWNERS | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cmd/clusterlink/OWNERS diff --git a/cmd/clusterlink/OWNERS b/cmd/clusterlink/OWNERS new file mode 100644 index 000000000..b79835439 --- /dev/null +++ b/cmd/clusterlink/OWNERS @@ -0,0 +1,10 @@ +approvers: + - wuyingjun-lucky + - hanweisen + - wangyizhi1 + - OrangeBao +reviewers: + - wuyingjun-lucky + - hanweisen + - wangyizhi1 + - OrangeBao \ No newline at end of file From f9c51a356906cfd7bca442c31edcc4ac355dd023 Mon Sep 17 00:00:00 2001 From: OrangeBao Date: Fri, 24 Nov 2023 10:48:05 +0800 Subject: [PATCH 5/8] fix: cannot create iptable chain in some environment Signed-off-by: OrangeBao --- deploy/clusterlink-agent.yaml | 15 +++++++++++++++ pkg/operator/clusterlink/agent/manifests.go | 14 +++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/deploy/clusterlink-agent.yaml b/deploy/clusterlink-agent.yaml index a56b6b00f..cde3bb24e 100644 --- a/deploy/clusterlink-agent.yaml +++ b/deploy/clusterlink-agent.yaml @@ -46,6 +46,12 @@ spec: - mountPath: /etc/clusterlink/kubeconfig name: proxy-config readOnly: true + - mountPath: /run/xtables.lock + name: iptableslock + readOnly: false + - mountPath: /lib/modules + name: lib-modules + readOnly: true terminationGracePeriodSeconds: 30 securityContext: privileged: true @@ -55,3 +61,12 @@ spec: configMap: defaultMode: 420 name: proxy-config + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: iptableslock + - name: lib-modules + hostPath: + path: /lib/modules + + diff --git a/pkg/operator/clusterlink/agent/manifests.go b/pkg/operator/clusterlink/agent/manifests.go index d8545423c..fdac753e0 100644 --- a/pkg/operator/clusterlink/agent/manifests.go +++ b/pkg/operator/clusterlink/agent/manifests.go @@ -56,13 +56,25 @@ spec: - mountPath: /etc/clusterlink name: proxy-config readOnly: true + - mountPath: /run/xtables.lock + name: iptableslock + readOnly: false + - mountPath: /lib/modules + name: lib-modules + readOnly: true terminationGracePeriodSeconds: 30 hostNetwork: true volumes: - name: proxy-config secret: secretName: {{ .ProxyConfigMapName }} - + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: iptableslock + - name: lib-modules + hostPath: + path: /lib/modules ` // DaemonSetReplace is a struct to help to concrete From 79fa5e342df7ffb480da1abb5aa8c1e6a858ecd4 Mon Sep 17 00:00:00 2001 From: wangyizhi1 Date: Fri, 24 Nov 2023 14:38:42 +0800 Subject: [PATCH 6/8] feat: add a way to exclude nodes Signed-off-by: wangyizhi1 --- deploy/clusterlink-elector.yml | 17 +++++++++++++++++ .../controllers/node/node_controller.go | 6 +++--- pkg/operator/clusterlink/elector/manifests.go | 6 ++++++ pkg/utils/constants.go | 2 ++ pkg/utils/k8s.go | 11 +++++++++++ 5 files changed, 39 insertions(+), 3 deletions(-) diff --git a/deploy/clusterlink-elector.yml b/deploy/clusterlink-elector.yml index 5b011b69f..42840cafe 100644 --- a/deploy/clusterlink-elector.yml +++ b/deploy/clusterlink-elector.yml @@ -21,6 +21,23 @@ spec: labels: app: elector spec: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kosmos.io/exclude + operator: DoesNotExist + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - elector + namespaces: + - kosmos-system + topologyKey: kubernetes.io/hostname containers: - command: - clusterlink-elector diff --git a/pkg/clusterlink/controllers/node/node_controller.go b/pkg/clusterlink/controllers/node/node_controller.go index 65cd51494..1d19fe89a 100644 --- a/pkg/clusterlink/controllers/node/node_controller.go +++ b/pkg/clusterlink/controllers/node/node_controller.go @@ -43,15 +43,15 @@ type Reconciler struct { var predicatesFunc = predicate.Funcs{ CreateFunc: func(createEvent event.CreateEvent) bool { node := createEvent.Object.(*corev1.Node) - return !utils.IsKosmosNode(node) + return !utils.IsKosmosNode(node) && !utils.IsExcludeNode(node) }, UpdateFunc: func(updateEvent event.UpdateEvent) bool { node := updateEvent.ObjectNew.(*corev1.Node) - return !utils.IsKosmosNode(node) + return !utils.IsKosmosNode(node) && !utils.IsExcludeNode(node) }, DeleteFunc: func(deleteEvent event.DeleteEvent) bool { node := deleteEvent.Object.(*corev1.Node) - return !utils.IsKosmosNode(node) + return !utils.IsKosmosNode(node) && !utils.IsExcludeNode(node) }, GenericFunc: func(genericEvent event.GenericEvent) bool { return false diff --git a/pkg/operator/clusterlink/elector/manifests.go b/pkg/operator/clusterlink/elector/manifests.go index e386f95cf..7f17b4e97 100644 --- a/pkg/operator/clusterlink/elector/manifests.go +++ b/pkg/operator/clusterlink/elector/manifests.go @@ -33,6 +33,12 @@ spec: spec: serviceAccountName: {{ .Name }} affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kosmos.io/exclude + operator: DoesNotExist podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: diff --git a/pkg/utils/constants.go b/pkg/utils/constants.go index 6b88f8539..86f30d126 100644 --- a/pkg/utils/constants.go +++ b/pkg/utils/constants.go @@ -76,6 +76,8 @@ const ( KosmosSelectorKey = "kosmos.io/cluster-selector" KosmosTrippedLabels = "kosmos-io/tripped" KosmosPvcLabelSelector = "kosmos-io/label-selector" + KosmosExcludeNodeLabel = "kosmos.io/exclude" + KosmosExcludeNodeValue = "true" // on resorce (pv, configmap, secret), represents which cluster this resource belongs to KosmosResourceOwnersAnnotations = "kosmos-io/cluster-owners" diff --git a/pkg/utils/k8s.go b/pkg/utils/k8s.go index 168a4918d..f22024a25 100644 --- a/pkg/utils/k8s.go +++ b/pkg/utils/k8s.go @@ -218,6 +218,17 @@ func IsKosmosNode(node *corev1.Node) bool { return labelVal == KosmosNodeValue } +func IsExcludeNode(node *corev1.Node) bool { + if node == nil { + return false + } + labelVal, exist := node.ObjectMeta.Labels[KosmosExcludeNodeLabel] + if !exist { + return false + } + return labelVal == KosmosExcludeNodeValue +} + func IsVirtualPod(pod *corev1.Pod) bool { if pod.Labels != nil && pod.Labels[KosmosPodLabel] == "true" { return true From 2d0b9bb2c8e8edd0e1fb2058e3e253b0fc171457 Mon Sep 17 00:00:00 2001 From: wangyizhi1 Date: Fri, 24 Nov 2023 15:14:21 +0800 Subject: [PATCH 7/8] fix: network null pointer exceptions Signed-off-by: wangyizhi1 --- pkg/apis/kosmos/v1alpha1/cluster_types.go | 14 +++++++---- .../kosmos/v1alpha1/zz_generated.deepcopy.go | 24 +++++++++++++++---- .../network-manager/handlers/pod_routes.go | 10 ++++++++ .../handlers/vxlocal_mac_cache.go | 4 ++++ pkg/generated/openapi/zz_generated.openapi.go | 8 ++----- pkg/kosmosctl/join/join.go | 4 ++-- 6 files changed, 48 insertions(+), 16 deletions(-) diff --git a/pkg/apis/kosmos/v1alpha1/cluster_types.go b/pkg/apis/kosmos/v1alpha1/cluster_types.go index 1d997455c..9a220cd8a 100644 --- a/pkg/apis/kosmos/v1alpha1/cluster_types.go +++ b/pkg/apis/kosmos/v1alpha1/cluster_types.go @@ -36,20 +36,20 @@ type ClusterSpec struct { ImageRepository string `json:"imageRepository,omitempty"` // +optional - ClusterLinkOptions ClusterLinkOptions `json:"clusterLinkOptions,omitempty"` + ClusterLinkOptions *ClusterLinkOptions `json:"clusterLinkOptions,omitempty"` // +optional - ClusterTreeOptions ClusterTreeOptions `json:"clusterTreeOptions,omitempty"` + ClusterTreeOptions *ClusterTreeOptions `json:"clusterTreeOptions,omitempty"` } type ClusterStatus struct { // ClusterLinkStatus contain the cluster network information // +optional - ClusterLinkStatus ClusterLinkStatus `json:"clusterLinkStatus,omitempty"` + ClusterLinkStatus *ClusterLinkStatus `json:"clusterLinkStatus,omitempty"` // ClusterTreeStatus contain the member cluster leafNode end status // +optional - ClusterTreeStatus ClusterTreeStatus `json:"clusterTreeStatus,omitempty"` + ClusterTreeStatus *ClusterTreeStatus `json:"clusterTreeStatus,omitempty"` } type ClusterLinkOptions struct { @@ -173,9 +173,15 @@ type ClusterList struct { } func (c *Cluster) IsP2P() bool { + if c.Spec.ClusterLinkOptions == nil { + return false + } return c.Spec.ClusterLinkOptions.NetworkType == NetworkTypeP2P } func (c *Cluster) IsGateway() bool { + if c.Spec.ClusterLinkOptions == nil { + return false + } return c.Spec.ClusterLinkOptions.NetworkType == NetWorkTypeGateWay } diff --git a/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go index d2c5c55de..129e3e3aa 100644 --- a/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go @@ -258,8 +258,16 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = make([]byte, len(*in)) copy(*out, *in) } - in.ClusterLinkOptions.DeepCopyInto(&out.ClusterLinkOptions) - in.ClusterTreeOptions.DeepCopyInto(&out.ClusterTreeOptions) + if in.ClusterLinkOptions != nil { + in, out := &in.ClusterLinkOptions, &out.ClusterLinkOptions + *out = new(ClusterLinkOptions) + (*in).DeepCopyInto(*out) + } + if in.ClusterTreeOptions != nil { + in, out := &in.ClusterTreeOptions, &out.ClusterTreeOptions + *out = new(ClusterTreeOptions) + (*in).DeepCopyInto(*out) + } return } @@ -276,8 +284,16 @@ func (in *ClusterSpec) DeepCopy() *ClusterSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterStatus) DeepCopyInto(out *ClusterStatus) { *out = *in - in.ClusterLinkStatus.DeepCopyInto(&out.ClusterLinkStatus) - in.ClusterTreeStatus.DeepCopyInto(&out.ClusterTreeStatus) + if in.ClusterLinkStatus != nil { + in, out := &in.ClusterLinkStatus, &out.ClusterLinkStatus + *out = new(ClusterLinkStatus) + (*in).DeepCopyInto(*out) + } + if in.ClusterTreeStatus != nil { + in, out := &in.ClusterTreeStatus, &out.ClusterTreeStatus + *out = new(ClusterTreeStatus) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/clusterlink/network-manager/handlers/pod_routes.go b/pkg/clusterlink/network-manager/handlers/pod_routes.go index 401ef059c..2d208f6df 100644 --- a/pkg/clusterlink/network-manager/handlers/pod_routes.go +++ b/pkg/clusterlink/network-manager/handlers/pod_routes.go @@ -143,7 +143,17 @@ func BuildRoutes(ctx *Context, target *v1alpha1.ClusterNode, cidrs []string) { } gw := ctx.Filter.GetGatewayNodeByClusterName(n.Spec.ClusterName) + if gw == nil { + klog.Warning("cannot find gateway node, cluster name: %s", n.Spec.ClusterName) + continue + } + gwDev := ctx.GetDeviceFromResults(gw.Name, vxLocal) + if gwDev == nil { + klog.Warning("cannot find the gw dev, nodeName: %s, devName: %s", gw.Name, vxLocal) + continue + } + gwIP, _, err := net.ParseCIDR(gwDev.Addr) if err != nil { klog.Warning("cannot parse gw dev addr, nodeName: %s, devName: %s", gw.Name, vxLocal) diff --git a/pkg/clusterlink/network-manager/handlers/vxlocal_mac_cache.go b/pkg/clusterlink/network-manager/handlers/vxlocal_mac_cache.go index c018eae20..1d589a3c1 100644 --- a/pkg/clusterlink/network-manager/handlers/vxlocal_mac_cache.go +++ b/pkg/clusterlink/network-manager/handlers/vxlocal_mac_cache.go @@ -20,6 +20,10 @@ func (h *VxLocalMacCache) Do(c *Context) (err error) { for _, node := range nodes { ipTypes := h.getSupportIPTypes(node, c) gw := c.Filter.GetGatewayNodeByClusterName(node.Spec.ClusterName) + if gw == nil { + klog.Warning("cannot find gateway node, cluster name: %s", node.Spec.ClusterName) + continue + } for _, ipType := range ipTypes { fdb, arp, err := h.buildVxLocalCachesByNode(c, ipType, gw) diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index d20237d68..d6d973bc8 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -575,14 +575,12 @@ func schema_pkg_apis_kosmos_v1alpha1_ClusterSpec(ref common.ReferenceCallback) c }, "clusterLinkOptions": { SchemaProps: spec.SchemaProps{ - Default: map[string]interface{}{}, - Ref: ref("github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.ClusterLinkOptions"), + Ref: ref("github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.ClusterLinkOptions"), }, }, "clusterTreeOptions": { SchemaProps: spec.SchemaProps{ - Default: map[string]interface{}{}, - Ref: ref("github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.ClusterTreeOptions"), + Ref: ref("github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.ClusterTreeOptions"), }, }, }, @@ -602,14 +600,12 @@ func schema_pkg_apis_kosmos_v1alpha1_ClusterStatus(ref common.ReferenceCallback) "clusterLinkStatus": { SchemaProps: spec.SchemaProps{ Description: "ClusterLinkStatus contain the cluster network information", - Default: map[string]interface{}{}, Ref: ref("github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.ClusterLinkStatus"), }, }, "clusterTreeStatus": { SchemaProps: spec.SchemaProps{ Description: "ClusterTreeStatus contain the member cluster leafNode end status", - Default: map[string]interface{}{}, Ref: ref("github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.ClusterTreeStatus"), }, }, diff --git a/pkg/kosmosctl/join/join.go b/pkg/kosmosctl/join/join.go index d09838dee..d8018a8e4 100644 --- a/pkg/kosmosctl/join/join.go +++ b/pkg/kosmosctl/join/join.go @@ -215,7 +215,7 @@ func (o *CommandJoinOptions) runCluster() error { Kubeconfig: o.KubeConfigStream, Namespace: o.Namespace, ImageRepository: o.ImageRegistry, - ClusterLinkOptions: v1alpha1.ClusterLinkOptions{ + ClusterLinkOptions: &v1alpha1.ClusterLinkOptions{ Enable: o.EnableLink, BridgeCIDRs: v1alpha1.VxlanCIDRs{ IP: "220.0.0.0/8", @@ -228,7 +228,7 @@ func (o *CommandJoinOptions) runCluster() error { NetworkType: v1alpha1.NetWorkTypeGateWay, IPFamily: v1alpha1.IPFamilyTypeIPV4, }, - ClusterTreeOptions: v1alpha1.ClusterTreeOptions{ + ClusterTreeOptions: &v1alpha1.ClusterTreeOptions{ Enable: o.EnableTree, }, }, From a0b8710be256541e025d23e93770fb3a49a59153 Mon Sep 17 00:00:00 2001 From: OrangeBao Date: Fri, 24 Nov 2023 15:22:40 +0800 Subject: [PATCH 8/8] fix: delete pod who is terminating Signed-off-by: OrangeBao --- .../controllers/pod/leaf_pod_controller.go | 9 +++++---- .../controllers/pod/root_pod_controller.go | 16 +++++++++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/pkg/clustertree/cluster-manager/controllers/pod/leaf_pod_controller.go b/pkg/clustertree/cluster-manager/controllers/pod/leaf_pod_controller.go index 7320e2747..9f90f5595 100644 --- a/pkg/clustertree/cluster-manager/controllers/pod/leaf_pod_controller.go +++ b/pkg/clustertree/cluster-manager/controllers/pod/leaf_pod_controller.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -38,7 +39,7 @@ func (r *LeafPodReconciler) Reconcile(ctx context.Context, request reconcile.Req if err := r.Get(ctx, request.NamespacedName, &pod); err != nil { if apierrors.IsNotFound(err) { // delete pod in root - if err := r.safeDeletePodInRootCluster(ctx, request); err != nil { + if err := DeletePodInRootCluster(ctx, request.NamespacedName, r.RootClient); err != nil { return reconcile.Result{RequeueAfter: LeafPodRequeueTime}, nil } return reconcile.Result{}, nil @@ -96,9 +97,9 @@ func NewLeafDeleteOption(pod *corev1.Pod) client.DeleteOption { } } -func (r *LeafPodReconciler) safeDeletePodInRootCluster(ctx context.Context, request reconcile.Request) error { +func DeletePodInRootCluster(ctx context.Context, rootnamespacedname types.NamespacedName, rootClient client.Client) error { rPod := corev1.Pod{} - err := r.RootClient.Get(ctx, request.NamespacedName, &rPod) + err := rootClient.Get(ctx, rootnamespacedname, &rPod) if err != nil { if apierrors.IsNotFound(err) { @@ -111,7 +112,7 @@ func (r *LeafPodReconciler) safeDeletePodInRootCluster(ctx context.Context, requ rPodCopy := rPod.DeepCopy() deleteOption := NewRootDeleteOption(rPodCopy) - if err := r.RootClient.Delete(ctx, rPodCopy, deleteOption); err != nil { + if err := rootClient.Delete(ctx, rPodCopy, deleteOption); err != nil { if !apierrors.IsNotFound(err) { return err } diff --git a/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go b/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go index f527f28cf..11b7bb3df 100644 --- a/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go +++ b/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go @@ -128,7 +128,7 @@ func (r *RootPodReconciler) Reconcile(ctx context.Context, request reconcile.Req // wait for leaf resource init return reconcile.Result{RequeueAfter: RootPodRequeueTime}, nil } - if err := r.DeletePodInLeafCluster(ctx, lr, request.NamespacedName); err != nil { + if err := r.DeletePodInLeafCluster(ctx, lr, request.NamespacedName, false); err != nil { klog.Errorf("delete pod in leaf error[1]: %v, %s", err, request.NamespacedName) return reconcile.Result{RequeueAfter: RootPodRequeueTime}, nil } @@ -185,7 +185,7 @@ func (r *RootPodReconciler) Reconcile(ctx context.Context, request reconcile.Req // delete pod in leaf if !rootpod.GetDeletionTimestamp().IsZero() { - if err := r.DeletePodInLeafCluster(ctx, lr, request.NamespacedName); err != nil { + if err := r.DeletePodInLeafCluster(ctx, lr, request.NamespacedName, true); err != nil { klog.Errorf("delete pod in leaf error[1]: %v, %s", err, request.NamespacedName) return reconcile.Result{RequeueAfter: RootPodRequeueTime}, nil } @@ -802,14 +802,21 @@ func (r *RootPodReconciler) UpdatePodInLeafCluster(ctx context.Context, lr *leaf return nil } -func (r *RootPodReconciler) DeletePodInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, rootnamespacedname types.NamespacedName) error { +func (r *RootPodReconciler) DeletePodInLeafCluster(ctx context.Context, lr *leafUtils.LeafResource, rootnamespacedname types.NamespacedName, cleanflag bool) error { klog.V(4).Infof("Deleting pod %v/%+v", rootnamespacedname.Namespace, rootnamespacedname.Name) leafPod := &corev1.Pod{} + cleanRootPodFunc := func() error { + return DeletePodInRootCluster(ctx, rootnamespacedname, r.Client) + } + err := lr.Client.Get(ctx, rootnamespacedname, leafPod) if err != nil { if errors.IsNotFound(err) { + if cleanflag { + return cleanRootPodFunc() + } return nil } return err @@ -825,6 +832,9 @@ func (r *RootPodReconciler) DeletePodInLeafCluster(ctx context.Context, lr *leaf if err != nil { if errors.IsNotFound(err) { klog.V(4).Infof("Tried to delete pod %s/%s, but it did not exist in the cluster", leafPod.Namespace, leafPod.Name) + if cleanflag { + return cleanRootPodFunc() + } return nil } return fmt.Errorf("could not delete pod: %v", err)