From 86b8122f72bc0d35a432093ca9204afa409c654f Mon Sep 17 00:00:00 2001 From: ONE7live Date: Wed, 20 Nov 2024 19:12:05 +0800 Subject: [PATCH] fix: update kosmosctl to support new version Signed-off-by: ONE7live --- pkg/kosmosctl/floater/analysis.go | 9 +- pkg/kosmosctl/floater/check.go | 78 ++- pkg/kosmosctl/install/install.go | 60 +- pkg/kosmosctl/join/join.go | 65 +- pkg/kosmosctl/manifest/manifest_configmaps.go | 2 +- pkg/kosmosctl/manifest/manifest_crds.go | 579 +++++++++++++++--- pkg/kosmosctl/manifest/manifest_daemonsets.go | 2 +- .../manifest/manifest_deployments.go | 4 +- .../manifest/manifest_serviceaccounts.go | 2 +- pkg/kosmosctl/uninstall/uninstall.go | 29 +- pkg/kosmosctl/unjoin/unjoin.go | 104 ++-- 11 files changed, 725 insertions(+), 209 deletions(-) diff --git a/pkg/kosmosctl/floater/analysis.go b/pkg/kosmosctl/floater/analysis.go index e187328a8..868a72848 100644 --- a/pkg/kosmosctl/floater/analysis.go +++ b/pkg/kosmosctl/floater/analysis.go @@ -56,9 +56,8 @@ type PrintAnalysisData struct { } func NewCmdAnalysis(f ctlutil.Factory) *cobra.Command { - o := &CommandAnalysisOptions{ - Version: version.GetReleaseVersion().PatchRelease(), - } + o := &CommandAnalysisOptions{} + cmd := &cobra.Command{ Use: "analysis", Short: i18n.T("Analysis network connectivity between Kosmos clusters"), @@ -98,6 +97,10 @@ func (o *CommandAnalysisOptions) Complete(f ctlutil.Factory) error { return fmt.Errorf("kosmosctl analysis complete error, generate dynamic client failed: %s", err) } + if len(o.Version) == 0 { + o.Version = version.GetReleaseVersion().PatchRelease() + } + af := NewAnalysisFloater(o) if err = af.completeFromKubeConfigPath(o.KubeConfig, o.Context); err != nil { return err diff --git a/pkg/kosmosctl/floater/check.go b/pkg/kosmosctl/floater/check.go index 457c45707..5ca6a1db0 100644 --- a/pkg/kosmosctl/floater/check.go +++ b/pkg/kosmosctl/floater/check.go @@ -1,12 +1,15 @@ package floater import ( + "context" "fmt" "os" "strconv" "github.com/olekukonko/tablewriter" "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" ctlutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/i18n" "k8s.io/kubectl/pkg/util/templates" @@ -59,9 +62,8 @@ type PrintCheckData struct { } func NewCmdCheck() *cobra.Command { - o := &CommandCheckOptions{ - Version: version.GetReleaseVersion().PatchRelease(), - } + o := &CommandCheckOptions{} + cmd := &cobra.Command{ Use: "check", Short: i18n.T("Check network connectivity between Kosmos clusters"), @@ -89,6 +91,7 @@ func NewCmdCheck() *cobra.Command { flags.StringVarP(&o.Namespace, "namespace", "n", utils.DefaultNamespace, "Kosmos namespace.") flags.StringVarP(&o.ImageRepository, "image-repository", "r", utils.DefaultImageRepository, "Image repository.") flags.StringVarP(&o.DstImageRepository, "dst-image-repository", "", "", "Destination cluster image repository.") + flags.StringVar(&o.Version, "version", "", "image version for pull images") flags.StringVar(&o.KubeConfig, "kubeconfig", "", "Absolute path to the host kubeconfig file.") flags.StringVar(&o.Context, "context", "", "The name of the kubeconfig context.") flags.StringVar(&o.SrcKubeConfig, "src-kubeconfig", "", "Absolute path to the source cluster kubeconfig file.") @@ -104,6 +107,10 @@ func NewCmdCheck() *cobra.Command { } func (o *CommandCheckOptions) Complete() error { + if len(o.Version) == 0 { + o.Version = version.GetReleaseVersion().PatchRelease() + } + if len(o.DstImageRepository) == 0 { o.DstImageRepository = o.ImageRepository } @@ -127,7 +134,70 @@ func (o *CommandCheckOptions) Complete() error { func (o *CommandCheckOptions) Validate() error { if len(o.Namespace) == 0 { - return fmt.Errorf("namespace must be specified") + return fmt.Errorf("kosmosctl check validate error, namespace must be specified") + } + + nodeRoleLabel := utils.NodeRoleLabel + "=" + utils.NodeRoleValue + srcConfig, err := utils.RestConfig(o.SrcKubeConfig, o.Context) + if err != nil { + return fmt.Errorf("kosmosctl check validate error, exception in kubeconfig verification of source cluster") + } + srcK8sClient, err := kubernetes.NewForConfig(srcConfig) + if err != nil { + return fmt.Errorf("kosmosctl check validate error, generate source K8s basic client failed: %v", err) + } + srcNodes, err := srcK8sClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: nodeRoleLabel, + }) + if err != nil { + return fmt.Errorf("kosmosctl check validate error, src cluster has no available nodes: %v", err) + } + for _, node := range srcNodes.Items { + labels := node.Labels + if value, exists := labels[utils.KosmosExcludeNodeLabel]; !exists || value != utils.KosmosExcludeNodeValue { + nodeCopy := node.DeepCopy() + if nodeCopy.Labels == nil { + nodeCopy.Labels = make(map[string]string) + } + nodeCopy.Labels[utils.KosmosExcludeNodeLabel] = utils.KosmosExcludeNodeValue + + _, err = srcK8sClient.CoreV1().Nodes().Update(context.TODO(), nodeCopy, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("kosmosctl check validate error, source cluster agent node label update exception: %v", err) + } + } + } + + if o.DstFloater != nil { + dstConfig, err := utils.RestConfig(o.DstKubeConfig, o.Context) + if err != nil { + return fmt.Errorf("kosmosctl check verify error, kubeconfig verification of the target cluster failed") + } + dstK8sClient, err := kubernetes.NewForConfig(dstConfig) + if err != nil { + return fmt.Errorf("kosmosctl check validate error, generate target K8s basic client failed: %v", err) + } + dstNodes, err := dstK8sClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: nodeRoleLabel, + }) + if err != nil { + return fmt.Errorf("kosmosctl check validate error, dst cluster has no available nodes: %v", err) + } + for _, node := range dstNodes.Items { + labels := node.Labels + if value, exists := labels[utils.KosmosExcludeNodeLabel]; !exists || value != utils.KosmosExcludeNodeValue { + nodeCopy := node.DeepCopy() + if nodeCopy.Labels == nil { + nodeCopy.Labels = make(map[string]string) + } + nodeCopy.Labels[utils.KosmosExcludeNodeLabel] = utils.KosmosExcludeNodeValue + + _, err = dstK8sClient.CoreV1().Nodes().Update(context.TODO(), nodeCopy, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("kosmosctl check validate error, target cluster agent node label update exception: %v", err) + } + } + } } return nil diff --git a/pkg/kosmosctl/install/install.go b/pkg/kosmosctl/install/install.go index d67e4f77e..fefe00409 100644 --- a/pkg/kosmosctl/install/install.go +++ b/pkg/kosmosctl/install/install.go @@ -292,11 +292,11 @@ func (o *CommandInstallOptions) runClusterlink() error { if err != nil { return err } - clusterlinkClusterNode, err := util.GenerateCustomResourceDefinition(manifest.ClusterNode, nil) + clusterlinkClusterNode, err := util.GenerateCustomResourceDefinition(manifest.ClusterlinkClusterNode, nil) if err != nil { return err } - clusterlinkNodeConfig, err := util.GenerateCustomResourceDefinition(manifest.NodeConfig, nil) + clusterlinkNodeConfig, err := util.GenerateCustomResourceDefinition(manifest.ClusterlinkNodeConfig, nil) if err != nil { return err } @@ -334,7 +334,7 @@ func (o *CommandInstallOptions) runClusterlink() error { } klog.Info("Deployment " + networkManagerDeploy.Name + " has been created.") - operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{ + operatorDeploy, err := util.GenerateDeployment(manifest.ClusterlinkOperatorDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, Version: o.Version, UseProxy: o.UseProxy, @@ -489,27 +489,6 @@ func (o *CommandInstallOptions) runClustertree() error { } klog.Info("Deployment clustertree-cluster-manager has been created.") - operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{ - Namespace: o.Namespace, - Version: o.Version, - UseProxy: o.UseProxy, - ImageRepository: o.ImageRegistry, - }) - if err != nil { - return fmt.Errorf("kosmosctl install operator run error, operator generate deployment failed: %s", err) - } - _, err = o.K8sClient.AppsV1().Deployments(operatorDeploy.Namespace).Get(context.TODO(), operatorDeploy.Name, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - err = o.createOperator() - if err != nil { - return err - } - } else { - return fmt.Errorf("kosmosctl install operator run error, operator get deployment failed: %s", err) - } - } - return nil } @@ -583,7 +562,7 @@ func (o *CommandInstallOptions) runScheduler() error { } klog.Infof("ConfigMap %s has been created.", scheduleConfigFile.Name) - hostkubeConfigMap := &corev1.ConfigMap{ + hostKubeConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: utils.HostKubeConfigName, Namespace: o.Namespace, @@ -597,7 +576,7 @@ func (o *CommandInstallOptions) runScheduler() error { }()), }, } - _, err = o.K8sClient.CoreV1().ConfigMaps(o.Namespace).Create(context.TODO(), hostkubeConfigMap, metav1.CreateOptions{}) + _, err = o.K8sClient.CoreV1().ConfigMaps(o.Namespace).Create(context.TODO(), hostKubeConfigMap, metav1.CreateOptions{}) if err != nil { if !apierrors.IsAlreadyExists(err) { return fmt.Errorf("kosmosctl install scheduler run error, configmap options failed: %v", err) @@ -605,6 +584,31 @@ func (o *CommandInstallOptions) runScheduler() error { } klog.Info("ConfigMap host-kubeconfig has been created.") + klog.Info("Attempting to create Kosmos-Scheduler CRDs...") + crds := apiextensionsv1.CustomResourceDefinitionList{} + schedulerCDP, err := util.GenerateCustomResourceDefinition(manifest.SchedulerClusterDistributionPolicies, manifest.CRDReplace{ + Namespace: o.Namespace, + }) + if err != nil { + return err + } + schedulerDP, err := util.GenerateCustomResourceDefinition(manifest.SchedulerDistributionPolicies, nil) + if err != nil { + return err + } + crds.Items = append(crds.Items, *schedulerCDP, *schedulerDP) + for i := range crds.Items { + _, err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), &crds.Items[i], metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + klog.Warningf("CRD %v is existed, creation process will skip", &crds.Items[i].Name) + continue + } + return fmt.Errorf("kosmosctl install scheduler run error, crd options failed: %v", err) + } + klog.Info("Create CRD " + crds.Items[i].Name + " successful.") + } + klog.Info("Start creating kosmos-scheduler Deployment...") schedulerDeploy, err := util.GenerateDeployment(manifest.SchedulerDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, @@ -631,7 +635,7 @@ func (o *CommandInstallOptions) runScheduler() error { func (o *CommandInstallOptions) createOperator() error { klog.Info("Start creating ClusterLink-Operator...") - operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{ + operatorDeploy, err := util.GenerateDeployment(manifest.ClusterlinkOperatorDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, Version: o.Version, UseProxy: o.UseProxy, @@ -685,7 +689,7 @@ func (o *CommandInstallOptions) createOperator() error { return fmt.Errorf("kosmosctl install operator run error, operator options clusterrolebinding failed: %s", err) } - operatorSA, err := util.GenerateServiceAccount(manifest.KosmosOperatorServiceAccount, manifest.ServiceAccountReplace{ + operatorSA, err := util.GenerateServiceAccount(manifest.ClusterlinkOperatorServiceAccount, manifest.ServiceAccountReplace{ Namespace: o.Namespace, }) if err != nil { diff --git a/pkg/kosmosctl/join/join.go b/pkg/kosmosctl/join/join.go index fd7edc540..2beb16776 100644 --- a/pkg/kosmosctl/join/join.go +++ b/pkg/kosmosctl/join/join.go @@ -301,6 +301,13 @@ func (o *CommandJoinOptions) CreateTreeRelatedCRDs() error { func (o *CommandJoinOptions) runCluster() error { klog.Info("Start registering cluster to kosmos control plane...") + // create ns if it does not exist + kosmosNS := &corev1.Namespace{} + kosmosNS.Name = o.Namespace + _, err := o.K8sClient.CoreV1().Namespaces().Create(context.TODO(), kosmosNS, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("kosmosctl join run error, create namespace failed: %s", err) + } // create cluster in control panel cluster := v1alpha1.Cluster{ @@ -354,38 +361,26 @@ func (o *CommandJoinOptions) runCluster() error { cluster.Spec.ClusterLinkOptions.DefaultNICName = o.DefaultNICName cluster.Spec.ClusterLinkOptions.CNI = o.CNI - } - - if o.EnableTree { - // create ClusterTree related crds - err := o.CreateTreeRelatedCRDs() - if err != nil { - return err - } - clusterPodConvert, err := util.GenerateCustomResourceDefinition(manifest.ClusterPodConvert, nil) + clusterlinkOperatorSA, err := util.GenerateServiceAccount(manifest.ClusterlinkOperatorServiceAccount, manifest.ServiceAccountReplace{ + Namespace: o.Namespace, + }) if err != nil { - return err + return fmt.Errorf("kosmosctl join run error, generate serviceaccount failed: %s", err) } - _, err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), clusterPodConvert, metav1.CreateOptions{}) - if err != nil { - if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl join run error, crd options failed: %v", err) - } + _, err = o.K8sClient.CoreV1().ServiceAccounts(clusterlinkOperatorSA.Namespace).Create(context.TODO(), clusterlinkOperatorSA, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("kosmosctl join run error, create serviceaccount failed: %s", err) } - klog.Info("Create CRD " + clusterPodConvert.Name + " successful.") + klog.Info("ServiceAccount " + clusterlinkOperatorSA.Name + " has been created.") + } - podConvert, err := util.GenerateCustomResourceDefinition(manifest.PodConvert, nil) + if o.EnableTree { + // create ClusterTree related crds + err := o.CreateTreeRelatedCRDs() if err != nil { return err } - _, err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), podConvert, metav1.CreateOptions{}) - if err != nil { - if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl join run error, crd options failed: %v", err) - } - } - klog.Info("Create CRD " + podConvert.Name + " successful.") if len(o.LeafModel) > 0 { switch o.LeafModel { @@ -425,20 +420,12 @@ func (o *CommandJoinOptions) runCluster() error { } } - _, err := o.KosmosClient.KosmosV1alpha1().Clusters().Create(context.TODO(), &cluster, metav1.CreateOptions{}) + _, err = o.KosmosClient.KosmosV1alpha1().Clusters().Create(context.TODO(), &cluster, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("kosmosctl join run error, create cluster failed: %s", err) } klog.Info("Cluster " + o.Name + " has been created.") - // create ns if it does not exist - kosmosNS := &corev1.Namespace{} - kosmosNS.Name = o.Namespace - _, err = o.K8sClient.CoreV1().Namespaces().Create(context.TODO(), kosmosNS, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl join run error, create namespace failed: %s", err) - } - // create rbac kosmosControlSA, err := util.GenerateServiceAccount(manifest.KosmosControlServiceAccount, manifest.ServiceAccountReplace{ Namespace: o.Namespace, @@ -490,18 +477,6 @@ func (o *CommandJoinOptions) runCluster() error { } klog.Info("ClusterRoleBinding " + kosmosCRB.Name + " has been created.") - kosmosOperatorSA, err := util.GenerateServiceAccount(manifest.KosmosOperatorServiceAccount, manifest.ServiceAccountReplace{ - Namespace: o.Namespace, - }) - if err != nil { - return fmt.Errorf("kosmosctl join run error, generate serviceaccount failed: %s", err) - } - _, err = o.K8sClient.CoreV1().ServiceAccounts(kosmosOperatorSA.Namespace).Create(context.TODO(), kosmosOperatorSA, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl join run error, create serviceaccount failed: %s", err) - } - klog.Info("ServiceAccount " + kosmosOperatorSA.Name + " has been created.") - // ToDo Wait for all services to be running klog.Info("Cluster [" + o.Name + "] registration successful.") diff --git a/pkg/kosmosctl/manifest/manifest_configmaps.go b/pkg/kosmosctl/manifest/manifest_configmaps.go index 3a921df9b..cdb81cf91 100644 --- a/pkg/kosmosctl/manifest/manifest_configmaps.go +++ b/pkg/kosmosctl/manifest/manifest_configmaps.go @@ -67,10 +67,10 @@ data: disabled: - name: "VolumeBinding" - name: "TaintToleration" + - name: "LeafNodeDistribution" enabled: - name: "LeafNodeTaintToleration" - name: "LeafNodeVolumeBinding" - - name: "LeafNodeDistribution" score: disabled: - name: "VolumeBinding" diff --git a/pkg/kosmosctl/manifest/manifest_crds.go b/pkg/kosmosctl/manifest/manifest_crds.go index 441b8c6a7..47d4706a8 100644 --- a/pkg/kosmosctl/manifest/manifest_crds.go +++ b/pkg/kosmosctl/manifest/manifest_crds.go @@ -300,81 +300,6 @@ spec: ` ) -const ClusterNode = `--- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.11.0 - creationTimestamp: null - name: clusternodes.kosmos.io -spec: - group: kosmos.io - names: - kind: ClusterNode - listKind: ClusterNodeList - plural: clusternodes - singular: clusternode - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .spec.roles - name: ROLES - type: string - - jsonPath: .spec.interfaceName - name: INTERFACE - type: string - - jsonPath: .spec.ip - name: IP - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - clusterName: - type: string - interfaceName: - type: string - ip: - type: string - ip6: - type: string - nodeName: - type: string - podCIDRs: - items: - type: string - type: array - roles: - items: - type: string - type: array - type: object - status: - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} -` - const Cluster = `--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -654,7 +579,83 @@ spec: storage: true subresources: {} ` -const NodeConfig = `--- + +const ClusterlinkClusterNode = `--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.0 + creationTimestamp: null + name: clusternodes.kosmos.io +spec: + group: kosmos.io + names: + kind: ClusterNode + listKind: ClusterNodeList + plural: clusternodes + singular: clusternode + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.roles + name: ROLES + type: string + - jsonPath: .spec.interfaceName + name: INTERFACE + type: string + - jsonPath: .spec.ip + name: IP + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + clusterName: + type: string + interfaceName: + type: string + ip: + type: string + ip6: + type: string + nodeName: + type: string + podCIDRs: + items: + type: string + type: array + roles: + items: + type: string + type: array + type: object + status: + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +` + +const ClusterlinkNodeConfig = `--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -4234,6 +4235,432 @@ spec: status: {} ` +const SchedulerClusterDistributionPolicies = ` +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.0 + creationTimestamp: null + name: clusterdistributionpolicies.kosmos.io +spec: + group: kosmos.io + names: + categories: + - kosmos-io + kind: ClusterDistributionPolicy + listKind: ClusterDistributionPolicyList + plural: clusterdistributionpolicies + shortNames: + - cdp + singular: clusterdistributionpolicy + scope: Cluster + versions: + - additionalPrinterColumns: + - description: CreationTimestamp is a timestamp representing the server time when + this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. + jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DistributionSpec represents the desired behavior of ClusterDistributionPolicy. + properties: + policyTerms: + description: PolicyTerms represents the rule for select nodes to distribute + resources. + items: + properties: + advancedTerm: + description: AdvancedTerm represents scheduling restrictions + to a certain set of nodes. + properties: + nodeName: + description: NodeName is a request to schedule this pod + onto a specific node. If it is non-empty, the scheduler + simply schedules this pod onto that node, assuming that + it fits resource requirements. + type: string + nodeSelector: + additionalProperties: + type: string + description: 'NodeSelector is a selector which must be true + for the pod to fit on a node. Selector which must match + a node''s labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + type: object + x-kubernetes-map-type: atomic + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple + using the matching operator . + properties: + effect: + description: Effect indicates the taint effect to + match. Empty means match all taint effects. When + specified, allowed values are NoSchedule, PreferNoSchedule + and NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. If + the key is empty, operator must be Exists; this + combination means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and Equal. + Defaults to Equal. Exists is equivalent to wildcard + for value, so that a pod can tolerate all taints + of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period + of time the toleration (which must be of effect + NoExecute, otherwise this field is ignored) tolerates + the taint. By default, it is not set, which means + tolerate the taint forever (do not evict). Zero + and negative values will be treated as 0 (evict + immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration + matches to. If the operator is Exists, the value + should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + name: + type: string + nodeType: + default: mix + description: NodeType declares the type for scheduling node. + Valid options are "host", "leaf", "mix", "adv". + enum: + - host + - leaf + - mix + - adv + type: string + required: + - name + type: object + minItems: 1 + type: array + resourceSelectors: + description: ResourceSelectors used to select resources and is required. + items: + description: ResourceSelector the resources will be selected. + properties: + labelSelector: + description: Filter resource by labelSelector If target resource + name is not empty, labelSelector will be ignored. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: Name of the target resource. Default is empty, + which means selecting all resources. + type: string + namePrefix: + description: NamePrefix the prefix of the target resource name + type: string + policyName: + description: Name of the Policy. + type: string + required: + - policyName + type: object + minItems: 1 + type: array + required: + - policyTerms + - resourceSelectors + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} + +` + +const SchedulerDistributionPolicies = ` +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.0 + creationTimestamp: null + name: distributionpolicies.kosmos.io +spec: + group: kosmos.io + names: + categories: + - kosmos-io + kind: DistributionPolicy + listKind: DistributionPolicyList + plural: distributionpolicies + shortNames: + - dp + singular: distributionpolicy + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: CreationTimestamp is a timestamp representing the server time when + this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. + jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DistributionSpec represents the desired behavior of DistributionPolicy. + properties: + policyTerms: + description: PolicyTerms represents the rule for select nodes to distribute + resources. + items: + properties: + advancedTerm: + description: AdvancedTerm represents scheduling restrictions + to a certain set of nodes. + properties: + nodeName: + description: NodeName is a request to schedule this pod + onto a specific node. If it is non-empty, the scheduler + simply schedules this pod onto that node, assuming that + it fits resource requirements. + type: string + nodeSelector: + additionalProperties: + type: string + description: 'NodeSelector is a selector which must be true + for the pod to fit on a node. Selector which must match + a node''s labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + type: object + x-kubernetes-map-type: atomic + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple + using the matching operator . + properties: + effect: + description: Effect indicates the taint effect to + match. Empty means match all taint effects. When + specified, allowed values are NoSchedule, PreferNoSchedule + and NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. If + the key is empty, operator must be Exists; this + combination means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and Equal. + Defaults to Equal. Exists is equivalent to wildcard + for value, so that a pod can tolerate all taints + of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period + of time the toleration (which must be of effect + NoExecute, otherwise this field is ignored) tolerates + the taint. By default, it is not set, which means + tolerate the taint forever (do not evict). Zero + and negative values will be treated as 0 (evict + immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration + matches to. If the operator is Exists, the value + should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + name: + type: string + nodeType: + default: mix + description: NodeType declares the type for scheduling node. + Valid options are "host", "leaf", "mix", "adv". + enum: + - host + - leaf + - mix + - adv + type: string + required: + - name + type: object + minItems: 1 + type: array + resourceSelectors: + description: ResourceSelectors used to select resources and is required. + items: + description: ResourceSelector the resources will be selected. + properties: + labelSelector: + description: Filter resource by labelSelector If target resource + name is not empty, labelSelector will be ignored. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: Name of the target resource. Default is empty, + which means selecting all resources. + type: string + namePrefix: + description: NamePrefix the prefix of the target resource name + type: string + policyName: + description: Name of the Policy. + type: string + required: + - policyName + type: object + minItems: 1 + type: array + required: + - policyTerms + - resourceSelectors + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} + +` + type CRDReplace struct { Namespace string } diff --git a/pkg/kosmosctl/manifest/manifest_daemonsets.go b/pkg/kosmosctl/manifest/manifest_daemonsets.go index 0fec93712..56c3bd513 100644 --- a/pkg/kosmosctl/manifest/manifest_daemonsets.go +++ b/pkg/kosmosctl/manifest/manifest_daemonsets.go @@ -30,7 +30,7 @@ spec: operator: DoesNotExist containers: - name: floater - image: {{ .ImageRepository }}/clusterlink-floater:v{{ .Version }} + image: {{ .ImageRepository }}/clusterlink-floater:{{ .Version }} imagePullPolicy: IfNotPresent command: - clusterlink-floater diff --git a/pkg/kosmosctl/manifest/manifest_deployments.go b/pkg/kosmosctl/manifest/manifest_deployments.go index 2d5d4da30..2aca37506 100644 --- a/pkg/kosmosctl/manifest/manifest_deployments.go +++ b/pkg/kosmosctl/manifest/manifest_deployments.go @@ -36,7 +36,7 @@ spec: memory: 500Mi ` - KosmosOperatorDeployment = ` + ClusterlinkOperatorDeployment = ` apiVersion: apps/v1 kind: Deployment metadata: @@ -94,7 +94,6 @@ spec: - name: proxy-config secret: secretName: controlpanel-config - ` ClusterTreeClusterManagerDeployment = `--- @@ -259,7 +258,6 @@ spec: path: customer-hosts name: coredns-customer-hosts name: customer-hosts - ` SchedulerDeployment = ` diff --git a/pkg/kosmosctl/manifest/manifest_serviceaccounts.go b/pkg/kosmosctl/manifest/manifest_serviceaccounts.go index 5cac44ffc..67c133941 100644 --- a/pkg/kosmosctl/manifest/manifest_serviceaccounts.go +++ b/pkg/kosmosctl/manifest/manifest_serviceaccounts.go @@ -9,7 +9,7 @@ metadata: namespace: {{ .Namespace }} ` - KosmosOperatorServiceAccount = ` + ClusterlinkOperatorServiceAccount = ` apiVersion: v1 kind: ServiceAccount metadata: diff --git a/pkg/kosmosctl/uninstall/uninstall.go b/pkg/kosmosctl/uninstall/uninstall.go index 777458fa5..35fee120b 100644 --- a/pkg/kosmosctl/uninstall/uninstall.go +++ b/pkg/kosmosctl/uninstall/uninstall.go @@ -211,7 +211,7 @@ func (o *CommandUninstallOptions) runClusterlink() error { } else if clusternodes != nil && len(clusternodes.Items) > 0 { klog.Info("kosmosctl uninstall warning, skip removing clusternode crd because cr instance exists") } else { - clusternodeCRD, err := util.GenerateCustomResourceDefinition(manifest.ClusterNode, nil) + clusternodeCRD, err := util.GenerateCustomResourceDefinition(manifest.ClusterlinkClusterNode, nil) if err != nil { return err } @@ -230,7 +230,7 @@ func (o *CommandUninstallOptions) runClusterlink() error { } else if nodeconfigs != nil && len(nodeconfigs.Items) > 0 { klog.Info("kosmosctl uninstall warning, skip removing nodeconfig crd because cr instance exists") } else { - nodeConfigCRD, err := util.GenerateCustomResourceDefinition(manifest.NodeConfig, nil) + nodeConfigCRD, err := util.GenerateCustomResourceDefinition(manifest.ClusterlinkNodeConfig, nil) if err != nil { return err } @@ -287,7 +287,7 @@ func (o *CommandUninstallOptions) runClusterlink() error { _, err = o.K8sClient.AppsV1().Deployments(o.Namespace).Get(context.Background(), clustertreeDeploy.Name, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { - operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{ + operatorDeploy, err := util.GenerateDeployment(manifest.ClusterlinkOperatorDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, Version: o.Version, }) @@ -419,7 +419,7 @@ func (o *CommandUninstallOptions) runClustertree() error { _, err = o.K8sClient.AppsV1().Deployments(o.Namespace).Get(context.Background(), clusterlinkDeploy.Name, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { - operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{ + operatorDeploy, err := util.GenerateDeployment(manifest.ClusterlinkOperatorDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, Version: o.Version, }) @@ -508,6 +508,27 @@ func (o *CommandUninstallOptions) runScheduler() error { klog.Info("ServiceAccount " + schedulerSA.Name + " is deleted.") } + clusterDPs, err := o.KosmosClient.KosmosV1alpha1().ClusterDistributionPolicies().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("kosmosctl uninstall kosmos-scheduler run error, list cluster distribution policy failed: %v", err) + } + } else if clusterDPs != nil && len(clusterDPs.Items) > 0 { + klog.Info("kosmosctl uninstall warning, skip removing cluster distribution policy crd because cr instance exists") + } else { + schedulerCDP, err := util.GenerateCustomResourceDefinition(manifest.SchedulerClusterDistributionPolicies, nil) + if err != nil { + return err + } + err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.Background(), schedulerCDP.Name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("kosmosctl uninstall kosmos-scheduler run error, cluster distribution policy crd delete failed: %v", err) + } + klog.Info("CRD " + schedulerCDP.Name + " is deleted.") + } + + // ToDo delete namespace level crd DistributionPolicy + klog.Info("Scheduler uninstalled.") return nil } diff --git a/pkg/kosmosctl/unjoin/unjoin.go b/pkg/kosmosctl/unjoin/unjoin.go index b859f386c..bbadc0217 100644 --- a/pkg/kosmosctl/unjoin/unjoin.go +++ b/pkg/kosmosctl/unjoin/unjoin.go @@ -6,6 +6,7 @@ import ( "time" "github.com/spf13/cobra" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" extensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -137,6 +138,13 @@ func (o *CommandUnJoinOptions) Run(args []string) error { func (o *CommandUnJoinOptions) runCluster() error { klog.Info("Start removing cluster from kosmos control plane...") + unjoinCluster, err := o.KosmosClient.KosmosV1alpha1().Clusters().Get(context.TODO(), o.Name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return fmt.Errorf("kosmosctl unjoin run error, get cluster failed: %s", err) + } + } + // delete cluster for { err := o.KosmosClient.KosmosV1alpha1().Clusters().Delete(context.TODO(), o.Name, metav1.DeleteOptions{}) @@ -150,30 +158,38 @@ func (o *CommandUnJoinOptions) runCluster() error { } klog.Info("Cluster: " + o.Name + " has been deleted.") - // delete crd - serviceExport, err := util.GenerateCustomResourceDefinition(manifest.ServiceExport, nil) - if err != nil { - return err - } - err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.Background(), serviceExport.Name, metav1.DeleteOptions{}) - if err != nil { - if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl unjoin run error, crd options failed: %v", err) + if unjoinCluster.Spec.ClusterTreeOptions.Enable { + // delete clustertree related crds + crds := apiextensionsv1.CustomResourceDefinitionList{} + serviceExport, err := util.GenerateCustomResourceDefinition(manifest.ServiceExport, nil) + if err != nil { + return err } - } - klog.Info("CRD: " + serviceExport.Name + " has been deleted.") - - serviceImport, err := util.GenerateCustomResourceDefinition(manifest.ServiceImport, nil) - if err != nil { - return err - } - err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.Background(), serviceImport.Name, metav1.DeleteOptions{}) - if err != nil { - if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl unjoin run error, crd options failed: %v", err) + serviceImport, err := util.GenerateCustomResourceDefinition(manifest.ServiceImport, nil) + if err != nil { + return err + } + clusterPodConvert, err := util.GenerateCustomResourceDefinition(manifest.ClusterPodConvert, nil) + if err != nil { + return err + } + podConvert, err := util.GenerateCustomResourceDefinition(manifest.PodConvert, nil) + if err != nil { + return err + } + crds.Items = append(crds.Items, *serviceImport, *serviceExport, *clusterPodConvert, *podConvert) + for i := range crds.Items { + err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.Background(), crds.Items[i].Name, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + klog.Warningf("CRD %v not found, deletion process will be skipped", crds.Items[i].Name) + continue + } + return fmt.Errorf("kosmosctl unjoin run error, crd options failed: %v", err) + } + klog.Info("CRD " + crds.Items[i].Name + " has been deleted.") } } - klog.Info("CRD: " + serviceImport.Name + " has been deleted.") // delete rbac err = o.K8sClient.CoreV1().Secrets(o.Namespace).Delete(context.TODO(), utils.ControlPanelSecretName, metav1.DeleteOptions{}) @@ -182,18 +198,6 @@ func (o *CommandUnJoinOptions) runCluster() error { } klog.Info("Secret: " + utils.ControlPanelSecretName + " has been deleted.") - err = o.K8sClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), utils.ExternalIPPoolNamePrefix, metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("kosmosctl unjoin run error, delete clusterrolebinding failed: %s", err) - } - klog.Info("ClusterRoleBinding: " + utils.ExternalIPPoolNamePrefix + " has been deleted.") - - err = o.K8sClient.RbacV1().ClusterRoles().Delete(context.TODO(), utils.ExternalIPPoolNamePrefix, metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("kosmosctl unjoin run error, delete clusterrole failed: %s", err) - } - klog.Info("ClusterRole: " + utils.ExternalIPPoolNamePrefix + " has been deleted.") - kosmosCR, err := util.GenerateClusterRole(manifest.KosmosClusterRole, nil) if err != nil { return fmt.Errorf("kosmosctl unjoin run error, generate clusterrole failed: %s", err) @@ -216,16 +220,6 @@ func (o *CommandUnJoinOptions) runCluster() error { } klog.Info("ClusterRoleBinding " + kosmosCRB.Name + " has been deleted.") - kosmosOperatorSA, err := util.GenerateServiceAccount(manifest.KosmosOperatorServiceAccount, nil) - if err != nil { - return err - } - err = o.K8sClient.CoreV1().ServiceAccounts(o.Namespace).Delete(context.TODO(), kosmosOperatorSA.Name, metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("kosmosctl unjoin run error, delete serviceaccout failed: %s", err) - } - klog.Info("ServiceAccount: " + kosmosOperatorSA.Name + " has been deleted.") - kosmosControlSA, err := util.GenerateServiceAccount(manifest.KosmosControlServiceAccount, manifest.ServiceAccountReplace{ Namespace: o.Namespace, }) @@ -238,6 +232,30 @@ func (o *CommandUnJoinOptions) runCluster() error { } klog.Info("ServiceAccount " + kosmosControlSA.Name + " has been deleted.") + if unjoinCluster.Spec.ClusterLinkOptions.Enable { + err = o.K8sClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), utils.ExternalIPPoolNamePrefix, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("kosmosctl unjoin run error, delete clusterrolebinding failed: %s", err) + } + klog.Info("ClusterRoleBinding: " + utils.ExternalIPPoolNamePrefix + " has been deleted.") + + err = o.K8sClient.RbacV1().ClusterRoles().Delete(context.TODO(), utils.ExternalIPPoolNamePrefix, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("kosmosctl unjoin run error, delete clusterrole failed: %s", err) + } + klog.Info("ClusterRole: " + utils.ExternalIPPoolNamePrefix + " has been deleted.") + + clusterlinkOperatorSA, err := util.GenerateServiceAccount(manifest.ClusterlinkOperatorServiceAccount, nil) + if err != nil { + return err + } + err = o.K8sClient.CoreV1().ServiceAccounts(o.Namespace).Delete(context.TODO(), clusterlinkOperatorSA.Name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("kosmosctl unjoin run error, delete serviceaccout failed: %s", err) + } + klog.Info("ServiceAccount: " + clusterlinkOperatorSA.Name + " has been deleted.") + } + // if cluster is not the master, delete namespace if o.Name != utils.DefaultClusterName { err = o.K8sClient.CoreV1().Namespaces().Delete(context.TODO(), o.Namespace, metav1.DeleteOptions{})