From 40620fc2d248aba99432608588a8551ff24f4afd Mon Sep 17 00:00:00 2001 From: duanmengkk Date: Thu, 14 Dec 2023 13:44:20 +0800 Subject: [PATCH] feat: make kosmosctl support --context Signed-off-by: duanmengkk --- hack/cluster.sh | 40 +++- hack/prepare-e2e.sh | 14 +- pkg/kosmosctl/floater/analysis.go | 4 +- pkg/kosmosctl/floater/check.go | 11 +- pkg/kosmosctl/floater/floater.go | 7 +- pkg/kosmosctl/install/install.go | 60 +++-- pkg/kosmosctl/join/join.go | 54 ++--- pkg/kosmosctl/kosmosctl.go | 6 +- .../manifest/manifest_deployments.go | 8 +- pkg/kosmosctl/rsmigrate/options.go | 24 +- pkg/kosmosctl/rsmigrate/serviceexport.go | 19 +- pkg/kosmosctl/rsmigrate/serviceimport.go | 12 +- pkg/kosmosctl/uninstall/uninstall.go | 41 ++-- pkg/kosmosctl/unjoin/unjoin.go | 29 +-- pkg/utils/k8s.go | 169 +------------- pkg/utils/kube_client.go | 211 ++++++++++++++++++ pkg/utils/node_client.go | 85 ------- test/e2e/deploy/cr/cluster-cr.yaml | 1 - test/e2e/deploy/nginx/nginx.yaml | 1 - test/e2e/leaf_node_test.go | 5 +- 20 files changed, 383 insertions(+), 418 deletions(-) create mode 100644 pkg/utils/kube_client.go delete mode 100644 pkg/utils/node_client.go diff --git a/hack/cluster.sh b/hack/cluster.sh index 6a04e1f5b..f45c8d1f8 100755 --- a/hack/cluster.sh +++ b/hack/cluster.sh @@ -17,6 +17,7 @@ CERT=$(cat ${ROOT}/pkg/cert/crt.pem | base64 -w 0) KEY=$(cat ${ROOT}/pkg/cert/key.pem | base64 -w 0) CN_ZONE=${CN_ZONE:-false} +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" if [ $REUSE == true ]; then echo "!!!!!!!!!!!Warning: Setting REUSE to true will not delete existing clusters.!!!!!!!!!!!" @@ -147,8 +148,6 @@ function create_cluster() { "kubectl get nodes | awk 'NR>1 {if (\$2 != \"Ready\") exit 1; }' && [ \$(kubectl get nodes --no-headers | wc -l) -eq ${N} ]" \ 300 echo "all node ready" - - kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/crds/mcs } function join_cluster() { @@ -185,11 +184,46 @@ EOF kubectl --context="kind-${member_cluster}" apply -f "$ROOT"/deploy/clusterlink-datapanel-rbac.yml } +function join_cluster_by_ctl(){ + local host_cluster=$1 + local member_cluster=$2 + HOST_CLUSTER_DIR="${ROOT}/environments/${host_cluster}" + MEMBER_CLUSTER_DIR="${ROOT}/environments/${member_cluster}" + kosmosctl join cluster --name $member_cluster --host-kubeconfig $HOST_CLUSTER_DIR/kubeconfig --kubeconfig $MEMBER_CLUSTER_DIR/kubeconfig --enable-all --version latest +} + + +function deploy_cluster_by_ctl(){ + local -r clustername=$1 + CLUSTER_DIR="${ROOT}/environments/${clustername}" + load_cluster_images "$clustername" + kosmosctl install --version latest --kubeconfig $CLUSTER_DIR/kubeconfig + + # deploy kosmos-scheduler for e2e test case of mysql-operator + sed -e "s|__VERSION__|$VERSION|g" -e "w ${ROOT}/environments/kosmos-scheduler.yml" "$ROOT"/deploy/scheduler/deployment.yaml + kubectl --context="kind-${clustername}" apply -f "${ROOT}/environments/kosmos-scheduler.yml" + kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/scheduler/rbac.yaml + + util::wait_for_condition "kosmos scheduler are ready" \ + "kubectl --context="kind-${clustername}" -n kosmos-system get deploy kosmos-scheduler -o jsonpath='{.status.replicas}{\" \"}{.status.readyReplicas}{\"\n\"}' | awk '{if (\$1 == \$2 && \$1 > 0) exit 0; else exit 1}'" \ + 300 + echo "cluster $clustername deploy kosmos-scheduler success" + + docker exec ${clustername}-control-plane /bin/sh -c "mv /etc/kubernetes/manifests/kube-scheduler.yaml /etc/kubernetes" + + # add the args for e2e test case of mysql-operator + kubectl --context="kind-${clustername}" -n kosmos-system patch deployment clustertree-cluster-manager --type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--auto-mcs-prefix=kosmos-e2e"}]' + + util::wait_for_condition "kosmos clustertree are ready" \ + "kubectl --context="kind-${clustername}" -n kosmos-system get deploy clustertree-cluster-manager -o jsonpath='{.status.replicas}{\" \"}{.status.readyReplicas}{\"\n\"}' | awk '{if (\$1 == \$2 && \$1 > 0) exit 0; else exit 1}'" \ + 300 +} + function deploy_cluster() { local -r clustername=$1 kubectl config use-context "kind-${clustername}" load_cluster_images "$clustername" - + kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/clusterlink-namespace.yml kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/kosmos-rbac.yml kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/crds diff --git a/hack/prepare-e2e.sh b/hack/prepare-e2e.sh index 763bcffc7..6e6ff8c64 100755 --- a/hack/prepare-e2e.sh +++ b/hack/prepare-e2e.sh @@ -4,8 +4,6 @@ set -o errexit set -o nounset set -o pipefail -ENABLED_MYSQL_E2E=${MYSQL_E2E:-true} - KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"} export KUBECONFIG=$KUBECONFIG_PATH/"config" @@ -27,16 +25,20 @@ source "$(dirname "${BASH_SOURCE[0]}")/install_kind_kubectl.sh" source "$(dirname "${BASH_SOURCE[0]}")/cluster.sh" make images GOOS="linux" --directory="${ROOT}" +make kosmosctl +os=$(go env GOOS) +arch=$(go env GOARCH) +export PATH=$PATH:"$ROOT"/_output/bin/"$os"/"$arch" + #cluster cluster create_cluster $HOST_CLUSTER_NAME $HOST_CLUSTER_POD_CIDR $HOST_CLUSTER_SERVICE_CIDR create_cluster $MEMBER1_CLUSTER_NAME $MEMBER1_CLUSTER_POD_CIDR $MEMBER1_CLUSTER_SERVICE_CIDR false create_cluster $MEMBER2_CLUSTER_NAME $MEMBER2_CLUSTER_POD_CIDR $MEMBER2_CLUSTER_SERVICE_CIDR fasle #deploy cluster -deploy_cluster $HOST_CLUSTER_NAME +deploy_cluster_by_ctl $HOST_CLUSTER_NAME load_cluster_images $MEMBER1_CLUSTER_NAME load_cluster_images $MEMBER2_CLUSTER_NAME #join cluster -join_cluster $HOST_CLUSTER_NAME $HOST_CLUSTER_NAME -join_cluster $HOST_CLUSTER_NAME $MEMBER1_CLUSTER_NAME -join_cluster $HOST_CLUSTER_NAME $MEMBER2_CLUSTER_NAME +join_cluster_by_ctl $HOST_CLUSTER_NAME $MEMBER1_CLUSTER_NAME +join_cluster_by_ctl $HOST_CLUSTER_NAME $MEMBER2_CLUSTER_NAME diff --git a/pkg/kosmosctl/floater/analysis.go b/pkg/kosmosctl/floater/analysis.go index b43ff7481..7c6b97e48 100644 --- a/pkg/kosmosctl/floater/analysis.go +++ b/pkg/kosmosctl/floater/analysis.go @@ -35,6 +35,7 @@ type CommandAnalysisOptions struct { ImageRepository string Version string KubeConfig string + Context string Port string PodWaitTime int @@ -78,6 +79,7 @@ func NewCmdAnalysis(f ctlutil.Factory) *cobra.Command { flags.StringVarP(&o.ImageRepository, "image-repository", "r", utils.DefaultImageRepository, "Image repository.") flags.StringVar(&o.Name, "name", "", "Specify the name of the resource to analysis.") flags.StringVar(&o.KubeConfig, "kubeconfig", "", "Absolute path to the cluster kubeconfig file.") + flags.StringVar(&o.Context, "context", "", "The name of the kubeconfig context.") flags.StringVar(&o.Port, "port", utils.DefaultPort, "Port used by floater.") flags.IntVarP(&o.PodWaitTime, "pod-wait-time", "w", utils.DefaultWaitTime, "Time for wait pod(floater) launch.") flags.BoolVar(&o.GenGraph, "gen-graph", false, "Configure generate network analysis graph.") @@ -97,7 +99,7 @@ func (o *CommandAnalysisOptions) Complete(f ctlutil.Factory) error { } af := NewAnalysisFloater(o) - if err = af.completeFromKubeConfigPath(o.KubeConfig); err != nil { + if err = af.completeFromKubeConfigPath(o.KubeConfig, o.Context); err != nil { return err } o.Floater = af diff --git a/pkg/kosmosctl/floater/check.go b/pkg/kosmosctl/floater/check.go index bc79898a0..457c45707 100644 --- a/pkg/kosmosctl/floater/check.go +++ b/pkg/kosmosctl/floater/check.go @@ -40,8 +40,12 @@ type CommandCheckOptions struct { HostNetwork bool KubeConfig string + Context string SrcKubeConfig string + SrcContext string + DstKubeConfig string + DstContext string SrcFloater *Floater DstFloater *Floater @@ -86,8 +90,11 @@ func NewCmdCheck() *cobra.Command { flags.StringVarP(&o.ImageRepository, "image-repository", "r", utils.DefaultImageRepository, "Image repository.") flags.StringVarP(&o.DstImageRepository, "dst-image-repository", "", "", "Destination cluster image repository.") flags.StringVar(&o.KubeConfig, "kubeconfig", "", "Absolute path to the host kubeconfig file.") + flags.StringVar(&o.Context, "context", "", "The name of the kubeconfig context.") flags.StringVar(&o.SrcKubeConfig, "src-kubeconfig", "", "Absolute path to the source cluster kubeconfig file.") + flags.StringVar(&o.SrcContext, "src-context", "", "The name of the src-kubeconfig context.") flags.StringVar(&o.DstKubeConfig, "dst-kubeconfig", "", "Absolute path to the destination cluster kubeconfig file.") + flags.StringVar(&o.DstContext, "dst-context", "", "The name of the dst-kubeconfig context.") flags.BoolVar(&o.HostNetwork, "host-network", false, "Configure HostNetwork.") flags.StringVar(&o.Port, "port", "8889", "Port used by floater.") flags.IntVarP(&o.PodWaitTime, "pod-wait-time", "w", 30, "Time for wait pod(floater) launch.") @@ -102,14 +109,14 @@ func (o *CommandCheckOptions) Complete() error { } srcFloater := NewCheckFloater(o, false) - if err := srcFloater.completeFromKubeConfigPath(o.SrcKubeConfig); err != nil { + if err := srcFloater.completeFromKubeConfigPath(o.SrcKubeConfig, o.SrcContext); err != nil { return err } o.SrcFloater = srcFloater if o.DstKubeConfig != "" { dstFloater := NewCheckFloater(o, true) - if err := dstFloater.completeFromKubeConfigPath(o.DstKubeConfig); err != nil { + if err := dstFloater.completeFromKubeConfigPath(o.DstKubeConfig, o.DstContext); err != nil { return err } o.DstFloater = dstFloater diff --git a/pkg/kosmosctl/floater/floater.go b/pkg/kosmosctl/floater/floater.go index 6466d3983..26b9138a1 100644 --- a/pkg/kosmosctl/floater/floater.go +++ b/pkg/kosmosctl/floater/floater.go @@ -14,7 +14,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/remotecommand" "k8s.io/klog/v2" @@ -102,13 +101,13 @@ func NewAnalysisFloater(o *CommandAnalysisOptions) *Floater { return floater } -func (f *Floater) completeFromKubeConfigPath(kubeConfigPath string) error { - config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) +func (f *Floater) completeFromKubeConfigPath(kubeConfigPath, context string) error { + config, err := utils.RestConfig(kubeConfigPath, context) if err != nil { return fmt.Errorf("kosmosctl docter complete error, generate floater config failed: %v", err) } - f.Config = config + f.Config = config f.Client, err = kubernetes.NewForConfig(f.Config) if err != nil { return fmt.Errorf("kosmosctl docter complete error, generate floater client failed: %v", err) diff --git a/pkg/kosmosctl/install/install.go b/pkg/kosmosctl/install/install.go index 6c7703e3b..7f4bdb979 100644 --- a/pkg/kosmosctl/install/install.go +++ b/pkg/kosmosctl/install/install.go @@ -3,8 +3,6 @@ package install import ( "context" "fmt" - "os" - "path/filepath" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" @@ -13,9 +11,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" "k8s.io/klog/v2" ctlutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/i18n" @@ -53,6 +49,7 @@ type CommandInstallOptions struct { Version string Module string HostKubeConfig string + HostContext string HostKubeConfigStream []byte WaitTime int @@ -71,7 +68,7 @@ type CommandInstallOptions struct { } // NewCmdInstall Install the Kosmos control plane in a Kubernetes cluster. -func NewCmdInstall(f ctlutil.Factory) *cobra.Command { +func NewCmdInstall() *cobra.Command { o := &CommandInstallOptions{} cmd := &cobra.Command{ @@ -82,7 +79,7 @@ func NewCmdInstall(f ctlutil.Factory) *cobra.Command { SilenceUsage: true, DisableFlagsInUseLine: true, RunE: func(cmd *cobra.Command, args []string) error { - ctlutil.CheckErr(o.Complete(f)) + ctlutil.CheckErr(o.Complete()) ctlutil.CheckErr(o.Validate()) ctlutil.CheckErr(o.Run()) return nil @@ -92,8 +89,10 @@ func NewCmdInstall(f ctlutil.Factory) *cobra.Command { flags := cmd.Flags() flags.StringVarP(&o.Namespace, "namespace", "n", utils.DefaultNamespace, "Kosmos namespace.") flags.StringVarP(&o.ImageRegistry, "private-image-registry", "", utils.DefaultImageRepository, "Private image registry where pull images from. If set, all required images will be downloaded from it, it would be useful in offline installation scenarios. In addition, you still can use --kube-image-registry to specify the registry for Kubernetes's images.") + flags.StringVar(&o.Version, "version", "", "image version for pull images") flags.StringVarP(&o.Module, "module", "m", utils.All, "Kosmos specify the module to install.") flags.StringVar(&o.HostKubeConfig, "kubeconfig", "", "Absolute path to the special kubeconfig file.") + flags.StringVar(&o.HostContext, "context", "", "The name of the kubeconfig context.") flags.StringVar(&o.CNI, "cni", "", "The cluster is configured using cni and currently supports calico and flannel.") flags.StringVar(&o.DefaultNICName, "default-nic", "", "Set default network interface card.") flags.StringVar(&o.NetworkType, "network-type", utils.NetworkTypeGateway, "Set the cluster network connection mode, which supports gateway and p2p modes, gateway is used by default.") @@ -107,35 +106,32 @@ func NewCmdInstall(f ctlutil.Factory) *cobra.Command { return cmd } -func (o *CommandInstallOptions) Complete(f ctlutil.Factory) error { - var config *rest.Config - var err error +func (o *CommandInstallOptions) Complete() error { + config, err := utils.RestConfig(o.HostKubeConfig, o.HostContext) + if err != nil { + return fmt.Errorf("kosmosctl install complete error, generate host config failed: %s", err) + } - if len(o.HostKubeConfig) > 0 { - config, err = clientcmd.BuildConfigFromFlags("", o.HostKubeConfig) - if err != nil { - return fmt.Errorf("kosmosctl install complete error, generate host config failed: %s", err) - } - o.HostKubeConfigStream, err = os.ReadFile(o.HostKubeConfig) - if err != nil { - return fmt.Errorf("kosmosctl install complete error, read host config failed: %s", err) - } - } else { - config, err = f.ToRESTConfig() - if err != nil { - return fmt.Errorf("kosmosctl install complete error, generate rest config failed: %v", err) - } - o.HostKubeConfigStream, err = os.ReadFile(filepath.Join(homedir.HomeDir(), ".kube", "config")) - if err != nil { - return fmt.Errorf("kosmosctl install complete error, read host config failed: %s", err) - } + rawConfig, err := utils.RawConfig(o.HostKubeConfig, o.HostContext) + if err != nil { + return fmt.Errorf("kosmosctl install complete error, generate host config failed: %s", err) + } + + streams, err := clientcmd.Write(rawConfig) + if err != nil { + return fmt.Errorf("kosmosctl install complete error, generate host config streams failed: %s", err) } + o.HostKubeConfigStream = streams o.KosmosClient, err = versioned.NewForConfig(config) if err != nil { return fmt.Errorf("kosmosctl install complete error, generate Kosmos client failed: %v", err) } + if o.Version == "" { + o.Version = fmt.Sprintf("v%s", version.GetReleaseVersion().PatchRelease()) + } + o.K8sClient, err = kubernetes.NewForConfig(config) if err != nil { return fmt.Errorf("kosmosctl install complete error, generate K8s basic client failed: %v", err) @@ -292,7 +288,7 @@ func (o *CommandInstallOptions) runClusterlink() error { networkManagerDeploy, err := util.GenerateDeployment(manifest.ClusterlinkNetworkManagerDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, ImageRepository: o.ImageRegistry, - Version: version.GetReleaseVersion().PatchRelease(), + Version: o.Version, }) if err != nil { return err @@ -312,7 +308,7 @@ func (o *CommandInstallOptions) runClusterlink() error { operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, - Version: version.GetReleaseVersion().PatchRelease(), + Version: o.Version, UseProxy: o.UseProxy, ImageRepository: o.ImageRegistry, }) @@ -443,7 +439,7 @@ func (o *CommandInstallOptions) runClustertree() error { clustertreeDeploy, err := util.GenerateDeployment(manifest.ClusterTreeClusterManagerDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, ImageRepository: o.ImageRegistry, - Version: version.GetReleaseVersion().PatchRelease(), + Version: o.Version, }) if err != nil { return err @@ -463,7 +459,7 @@ func (o *CommandInstallOptions) runClustertree() error { operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, - Version: version.GetReleaseVersion().PatchRelease(), + Version: o.Version, UseProxy: o.UseProxy, ImageRepository: o.ImageRegistry, }) @@ -489,7 +485,7 @@ func (o *CommandInstallOptions) createOperator() error { klog.Info("Start creating Kosmos-Operator...") operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, - Version: version.GetReleaseVersion().PatchRelease(), + Version: o.Version, UseProxy: o.UseProxy, ImageRepository: o.ImageRegistry, }) diff --git a/pkg/kosmosctl/join/join.go b/pkg/kosmosctl/join/join.go index d8018a8e4..3cbc91167 100644 --- a/pkg/kosmosctl/join/join.go +++ b/pkg/kosmosctl/join/join.go @@ -3,8 +3,6 @@ package join import ( "context" "fmt" - "os" - "path/filepath" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" @@ -12,9 +10,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" "k8s.io/klog/v2" ctlutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/i18n" @@ -25,6 +21,7 @@ import ( "github.com/kosmos.io/kosmos/pkg/kosmosctl/manifest" "github.com/kosmos.io/kosmos/pkg/kosmosctl/util" "github.com/kosmos.io/kosmos/pkg/utils" + "github.com/kosmos.io/kosmos/pkg/version" ) var joinExample = templates.Examples(i18n.T(` @@ -45,9 +42,12 @@ type CommandJoinOptions struct { Name string Namespace string ImageRegistry string + Version string KubeConfig string + Context string KubeConfigStream []byte HostKubeConfig string + HostContext string HostKubeConfigStream []byte WaitTime int RootFlag bool @@ -91,8 +91,11 @@ func NewCmdJoin(f ctlutil.Factory) *cobra.Command { flags.StringVar(&o.Name, "name", "", "Specify the name of the resource to join.") flags.StringVarP(&o.Namespace, "namespace", "n", utils.DefaultNamespace, "Kosmos namespace.") flags.StringVar(&o.KubeConfig, "kubeconfig", "", "Absolute path to the cluster kubeconfig file.") + flags.StringVar(&o.Context, "context", "", "The name of the kubeconfig context.") flags.StringVar(&o.HostKubeConfig, "host-kubeconfig", "", "Absolute path to the special host kubeconfig file.") + flags.StringVar(&o.HostContext, "host-context", "", "The name of the host-kubeconfig context.") flags.StringVar(&o.ImageRegistry, "private-image-registry", utils.DefaultImageRepository, "Private image registry where pull images from. If set, all required images will be downloaded from it, it would be useful in offline installation scenarios.") + flags.StringVar(&o.Version, "version", "", "image version for pull images") flags.BoolVar(&o.RootFlag, "root-flag", false, "Tag control cluster.") flags.BoolVar(&o.EnableAll, "enable-all", false, "Turn on all module.") flags.BoolVar(&o.EnableLink, "enable-link", false, "Turn on clusterlink.") @@ -109,46 +112,37 @@ func NewCmdJoin(f ctlutil.Factory) *cobra.Command { } func (o *CommandJoinOptions) Complete(f ctlutil.Factory) error { - var hostConfig *rest.Config - var clusterConfig *rest.Config - var err error + hostConfig, err := utils.RestConfig(o.HostKubeConfig, o.HostContext) + if err != nil { + return fmt.Errorf("kosmosctl join complete error, generate host rest config failed: %s", err) + } - if len(o.HostKubeConfig) > 0 { - hostConfig, err = clientcmd.BuildConfigFromFlags("", o.HostKubeConfig) - if err != nil { - return fmt.Errorf("kosmosctl join complete error, generate hostConfig failed: %s", err) - } - o.HostKubeConfigStream, err = os.ReadFile(o.HostKubeConfig) - if err != nil { - return fmt.Errorf("kosmosctl join complete error, read hostConfig failed: %s", err) - } - } else { - hostConfig, err = f.ToRESTConfig() - if err != nil { - return fmt.Errorf("kosmosctl join complete error, generate hostConfig failed: %s", err) - } - o.HostKubeConfigStream, err = os.ReadFile(filepath.Join(homedir.HomeDir(), ".kube", "config")) - if err != nil { - return fmt.Errorf("kosmosctl join complete error, read hostConfig failed: %s", err) - } + if o.Version == "" { + o.Version = fmt.Sprintf("v%s", version.GetReleaseVersion().PatchRelease()) } o.KosmosClient, err = versioned.NewForConfig(hostConfig) if err != nil { - return fmt.Errorf("kosmosctl install complete error, generate Kosmos client failed: %v", err) + return fmt.Errorf("kosmosctl join complete error, generate Kosmos client failed: %v", err) } if len(o.KubeConfig) > 0 { - o.KubeConfigStream, err = os.ReadFile(o.KubeConfig) + clusterConfig, err := utils.RestConfig(o.KubeConfig, o.Context) + if err != nil { + return fmt.Errorf("kosmosctl join complete error, generate rest config failed: %s", err) + } + + rawConfig, err := utils.RawConfig(o.KubeConfig, o.Context) if err != nil { - return fmt.Errorf("kosmosctl join run error, read KubeConfigStream failed: %s", err) + return fmt.Errorf("kosmosctl join complete error, generate raw config failed: %s", err) } - clusterConfig, err = clientcmd.BuildConfigFromFlags("", o.KubeConfig) + streams, err := clientcmd.Write(rawConfig) if err != nil { - return fmt.Errorf("kosmosctl join complete error, generate clusterConfig failed: %s", err) + return fmt.Errorf("kosmosctl join complete error, wite restconfig to streams failed: %s", err) } + o.KubeConfigStream = streams o.K8sClient, err = kubernetes.NewForConfig(clusterConfig) if err != nil { return fmt.Errorf("kosmosctl join complete error, generate K8s basic client failed: %v", err) diff --git a/pkg/kosmosctl/kosmosctl.go b/pkg/kosmosctl/kosmosctl.go index b28a09630..6fdce81d3 100644 --- a/pkg/kosmosctl/kosmosctl.go +++ b/pkg/kosmosctl/kosmosctl.go @@ -59,8 +59,8 @@ func NewKosmosCtlCommand() *cobra.Command { { Message: "Install/UnInstall Commands:", Commands: []*cobra.Command{ - install.NewCmdInstall(f), - uninstall.NewCmdUninstall(f), + install.NewCmdInstall(), + uninstall.NewCmdUninstall(), }, }, { @@ -81,7 +81,7 @@ func NewKosmosCtlCommand() *cobra.Command { Message: "Cluster Resource Import/Export Commands:", Commands: []*cobra.Command{ rsmigrate.NewCmdImport(f), - rsmigrate.NewCmdExport(f), + rsmigrate.NewCmdExport(), }, }, { diff --git a/pkg/kosmosctl/manifest/manifest_deployments.go b/pkg/kosmosctl/manifest/manifest_deployments.go index f888726b4..864e6e1cf 100644 --- a/pkg/kosmosctl/manifest/manifest_deployments.go +++ b/pkg/kosmosctl/manifest/manifest_deployments.go @@ -22,7 +22,7 @@ spec: serviceAccountName: clusterlink-network-manager containers: - name: manager - image: {{ .ImageRepository }}/clusterlink-network-manager:v{{ .Version }} + image: {{ .ImageRepository }}/clusterlink-network-manager:{{ .Version }} imagePullPolicy: IfNotPresent command: - clusterlink-network-manager @@ -69,7 +69,7 @@ spec: topologyKey: kubernetes.io/hostname containers: - name: operator - image: {{ .ImageRepository }}/kosmos-operator:v{{ .Version }} + image: {{ .ImageRepository }}/kosmos-operator:{{ .Version }} imagePullPolicy: IfNotPresent command: - kosmos-operator @@ -83,7 +83,7 @@ spec: memory: 200Mi env: - name: VERSION - value: v{{ .Version }} + value: {{ .Version }} - name: USE_PROXY value: "{{ .UseProxy }}" volumeMounts: @@ -118,7 +118,7 @@ spec: serviceAccountName: clustertree containers: - name: manager - image: {{ .ImageRepository }}/clustertree-cluster-manager:v{{ .Version }} + image: {{ .ImageRepository }}/clustertree-cluster-manager:{{ .Version }} imagePullPolicy: IfNotPresent env: - name: APISERVER_CERT_LOCATION diff --git a/pkg/kosmosctl/rsmigrate/options.go b/pkg/kosmosctl/rsmigrate/options.go index 58656f3a4..6af36a7c4 100644 --- a/pkg/kosmosctl/rsmigrate/options.go +++ b/pkg/kosmosctl/rsmigrate/options.go @@ -2,13 +2,9 @@ package rsmigrate import ( "fmt" - "os" - "path/filepath" "github.com/spf13/cobra" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/util/homedir" - ctlutil "k8s.io/kubectl/pkg/cmd/util" "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" kosmosversioned "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" @@ -25,6 +21,7 @@ type LeafClusterOptions struct { type CommandOptions struct { MasterKubeConfig string + MasterContext string MasterClient kubernetes.Interface //clientset operate leafCluster releted resource MasterKosmosClient kosmosversioned.Interface @@ -33,30 +30,19 @@ type CommandOptions struct { Namespace string } -func (o *CommandOptions) Validate(cmd *cobra.Command) error { - return nil -} - -func (o *CommandOptions) Complete(f ctlutil.Factory, cmd *cobra.Command) error { - var err error - var kubeConfigStream []byte - // get master kubernetes clientset - if len(o.MasterKubeConfig) > 0 { - kubeConfigStream, err = os.ReadFile(o.MasterKubeConfig) - } else { - kubeConfigStream, err = os.ReadFile(filepath.Join(homedir.HomeDir(), ".kube", "config")) - } +func (o *CommandOptions) Complete(cmd *cobra.Command) error { + config, err := utils.RestConfig(o.MasterKubeConfig, o.MasterContext) if err != nil { return fmt.Errorf("get master kubeconfig failed: %s", err) } - masterClient, err := utils.NewClientFromBytes(kubeConfigStream) + masterClient, err := kubernetes.NewForConfig(config) if err != nil { return fmt.Errorf("create master clientset error: %s ", err) } o.MasterClient = masterClient - kosmosClient, err := utils.NewKosmosClientFromBytes(kubeConfigStream) + kosmosClient, err := kosmosversioned.NewForConfig(config) if err != nil { return fmt.Errorf("get master rest client config error:%s", err) } diff --git a/pkg/kosmosctl/rsmigrate/serviceexport.go b/pkg/kosmosctl/rsmigrate/serviceexport.go index edc67ef2e..b5cd15311 100644 --- a/pkg/kosmosctl/rsmigrate/serviceexport.go +++ b/pkg/kosmosctl/rsmigrate/serviceexport.go @@ -24,7 +24,7 @@ type CommandExportOptions struct { } // NewCmdExport export resource to control plane -func NewCmdExport(f ctlutil.Factory) *cobra.Command { +func NewCmdExport() *cobra.Command { o := &CommandExportOptions{CommandOptions: &CommandOptions{SrcLeafClusterOptions: &LeafClusterOptions{}}} cmd := &cobra.Command{ @@ -35,36 +35,27 @@ func NewCmdExport(f ctlutil.Factory) *cobra.Command { SilenceUsage: true, DisableFlagsInUseLine: true, RunE: func(cmd *cobra.Command, args []string) error { - ctlutil.CheckErr(o.Complete(f, cmd)) - ctlutil.CheckErr(o.Validate(cmd)) + ctlutil.CheckErr(o.Complete(cmd)) ctlutil.CheckErr(o.Run(cmd, args)) return nil }, } cmd.Flags().StringVarP(&o.MasterKubeConfig, "kubeconfig", "", "", "Absolute path to the master kubeconfig file.") + cmd.Flags().StringVar(&o.MasterContext, "context", "", "The name of the kubeconfig context.") cmd.Flags().StringVarP(&o.Namespace, "namespace", "n", "default", "The namespace scope for this CLI request") return cmd } -func (o *CommandExportOptions) Complete(f ctlutil.Factory, cmd *cobra.Command) error { - err := o.CommandOptions.Complete(f, cmd) +func (o *CommandExportOptions) Complete(cmd *cobra.Command) error { + err := o.CommandOptions.Complete(cmd) if err != nil { return err } return nil } -func (o *CommandExportOptions) Validate(cmd *cobra.Command) error { - err := o.CommandOptions.Validate(cmd) - if err != nil { - return fmt.Errorf("%s, valid args error: %s", exportErr, err) - } - - return nil -} - func (o *CommandExportOptions) Run(cmd *cobra.Command, args []string) error { if len(args) == 0 { return fmt.Errorf("args is null, resource type should be specified") diff --git a/pkg/kosmosctl/rsmigrate/serviceimport.go b/pkg/kosmosctl/rsmigrate/serviceimport.go index b647ef4d4..b73ddbb77 100644 --- a/pkg/kosmosctl/rsmigrate/serviceimport.go +++ b/pkg/kosmosctl/rsmigrate/serviceimport.go @@ -40,21 +40,22 @@ func NewCmdImport(f ctlutil.Factory) *cobra.Command { SilenceUsage: true, DisableFlagsInUseLine: true, RunE: func(cmd *cobra.Command, args []string) error { - ctlutil.CheckErr(o.Complete(f, cmd)) + ctlutil.CheckErr(o.Complete(cmd)) ctlutil.CheckErr(o.Validate(cmd)) ctlutil.CheckErr(o.Run(f, cmd, args)) return nil }, } cmd.Flags().StringVarP(&o.MasterKubeConfig, "kubeconfig", "", "", "Absolute path to the master kubeconfig file.") + cmd.Flags().StringVar(&o.MasterContext, "context", "", "The name of the kubeconfig context.") cmd.Flags().StringVarP(&o.Namespace, "namespace", "n", "default", "The namespace scope for this CLI request") cmd.Flags().StringVar(&o.DstLeafClusterOptions.LeafClusterName, "to-leafcluster", "", "Import resource to this destination leafcluster") return cmd } -func (o *CommandImportOptions) Complete(f ctlutil.Factory, cmd *cobra.Command) error { - err := o.CommandOptions.Complete(f, cmd) +func (o *CommandImportOptions) Complete(cmd *cobra.Command) error { + err := o.CommandOptions.Complete(cmd) if err != nil { return err } @@ -71,11 +72,6 @@ func (o *CommandImportOptions) Complete(f ctlutil.Factory, cmd *cobra.Command) e } func (o *CommandImportOptions) Validate(cmd *cobra.Command) error { - err := o.CommandOptions.Validate(cmd) - if err != nil { - return fmt.Errorf("%s, valid args error: %s", importErr, err) - } - if !cmd.Flags().Changed("to-leafcluster") { return fmt.Errorf("%s, required flag(s) 'to-leafcluster' not set", importErr) } diff --git a/pkg/kosmosctl/uninstall/uninstall.go b/pkg/kosmosctl/uninstall/uninstall.go index 97930452b..a64c9089d 100644 --- a/pkg/kosmosctl/uninstall/uninstall.go +++ b/pkg/kosmosctl/uninstall/uninstall.go @@ -10,8 +10,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" ctlutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/i18n" @@ -21,6 +19,7 @@ import ( "github.com/kosmos.io/kosmos/pkg/kosmosctl/manifest" "github.com/kosmos.io/kosmos/pkg/kosmosctl/util" "github.com/kosmos.io/kosmos/pkg/utils" + "github.com/kosmos.io/kosmos/pkg/version" ) var uninstallExample = templates.Examples(i18n.T(` @@ -44,15 +43,18 @@ type CommandUninstallOptions struct { Namespace string Module string KubeConfig string + Context string KosmosClient versioned.Interface K8sClient kubernetes.Interface K8sDynamicClient *dynamic.DynamicClient K8sExtensionsClient extensionsclient.Interface + + Version string } // NewCmdUninstall Uninstall the Kosmos control plane in a Kubernetes cluster. -func NewCmdUninstall(f ctlutil.Factory) *cobra.Command { +func NewCmdUninstall() *cobra.Command { o := &CommandUninstallOptions{} cmd := &cobra.Command{ @@ -63,7 +65,7 @@ func NewCmdUninstall(f ctlutil.Factory) *cobra.Command { SilenceUsage: true, DisableFlagsInUseLine: true, RunE: func(cmd *cobra.Command, args []string) error { - ctlutil.CheckErr(o.Complete(f)) + ctlutil.CheckErr(o.Complete()) ctlutil.CheckErr(o.Validate()) ctlutil.CheckErr(o.Run()) return nil @@ -74,29 +76,24 @@ func NewCmdUninstall(f ctlutil.Factory) *cobra.Command { flags.StringVarP(&o.Namespace, "namespace", "n", utils.DefaultNamespace, "Kosmos namespace.") flags.StringVarP(&o.Module, "module", "m", utils.All, "Kosmos specify the module to uninstall.") flags.StringVar(&o.KubeConfig, "kubeconfig", "", "Absolute path to the special kubeconfig file.") - + flags.StringVar(&o.Context, "context", "", "The name of the kubeconfig context.") + flags.StringVar(&o.Version, "version", "", "image version for uninstall images") return cmd } -func (o *CommandUninstallOptions) Complete(f ctlutil.Factory) error { - var config *rest.Config - var err error +func (o *CommandUninstallOptions) Complete() error { + config, err := utils.RestConfig(o.KubeConfig, o.Context) + if err != nil { + return fmt.Errorf("kosmosctl uninstall complete error, generate config failed: %s", err) + } - if len(o.KubeConfig) > 0 { - config, err = clientcmd.BuildConfigFromFlags("", o.KubeConfig) - if err != nil { - return fmt.Errorf("kosmosctl uninstall complete error, generate config failed: %s", err) - } - } else { - config, err = f.ToRESTConfig() - if err != nil { - return fmt.Errorf("kosmosctl uninstall complete error, generate rest config failed: %v", err) - } + if o.Version == "" { + o.Version = fmt.Sprintf("v%s", version.GetReleaseVersion().PatchRelease()) } o.KosmosClient, err = versioned.NewForConfig(config) if err != nil { - return fmt.Errorf("kosmosctl install complete error, generate Kosmos client failed: %v", err) + return fmt.Errorf("kosmosctl uninstall complete error, generate Kosmos client failed: %v", err) } o.K8sClient, err = kubernetes.NewForConfig(config) @@ -165,6 +162,7 @@ func (o *CommandUninstallOptions) runClusterlink() error { klog.Info("Start uninstalling clusterlink from kosmos control plane...") clusterlinkDeploy, err := util.GenerateDeployment(manifest.ClusterlinkNetworkManagerDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, + Version: o.Version, }) if err != nil { return err @@ -283,6 +281,7 @@ func (o *CommandUninstallOptions) runClusterlink() error { if apierrors.IsNotFound(err) { operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, + Version: o.Version, }) if err != nil { return fmt.Errorf("kosmosctl uninstall clusterlink run error, generate operator deployment failed: %s", err) @@ -308,6 +307,7 @@ func (o *CommandUninstallOptions) runClustertree() error { klog.Info("Start uninstalling clustertree from kosmos control plane...") clustertreeDeploy, err := util.GenerateDeployment(manifest.ClusterTreeClusterManagerDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, + Version: o.Version, }) if err != nil { return err @@ -413,6 +413,7 @@ func (o *CommandUninstallOptions) runClustertree() error { if apierrors.IsNotFound(err) { operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, + Version: o.Version, }) if err != nil { return fmt.Errorf("kosmosctl uninstall clustertree run error, generate operator deployment failed: %s", err) @@ -509,7 +510,7 @@ func (o *CommandUninstallOptions) runCoredns() error { } err = o.K8sClient.RbacV1().ClusterRoles().Delete(context.TODO(), cRole.Name, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("kosmosctl install coredns run error, clusterrole options failed: %v", err) + return fmt.Errorf("kosmosctl uninstall coredns run error, clusterrole options failed: %v", err) } klog.Info("ClusterRole " + cRole.Name + " is deleted.") diff --git a/pkg/kosmosctl/unjoin/unjoin.go b/pkg/kosmosctl/unjoin/unjoin.go index 24ed7fcfc..19a7943c9 100644 --- a/pkg/kosmosctl/unjoin/unjoin.go +++ b/pkg/kosmosctl/unjoin/unjoin.go @@ -35,7 +35,9 @@ type CommandUnJoinOptions struct { Name string Namespace string KubeConfig string + Context string HostKubeConfig string + HostContext string KosmosClient versioned.Interface K8sClient kubernetes.Interface @@ -64,36 +66,29 @@ func NewCmdUnJoin(f ctlutil.Factory) *cobra.Command { cmd.Flags().StringVar(&o.Name, "name", "", "Specify the name of the resource to unjoin.") cmd.Flags().StringVarP(&o.Namespace, "namespace", "n", utils.DefaultNamespace, "Kosmos namespace.") cmd.Flags().StringVar(&o.KubeConfig, "kubeconfig", "", "Absolute path to the cluster kubeconfig file.") + cmd.Flags().StringVar(&o.Context, "context", "", "The name of the kubeconfig context.") cmd.Flags().StringVar(&o.HostKubeConfig, "host-kubeconfig", "", "Absolute path to the special host kubeconfig file.") + cmd.Flags().StringVar(&o.HostContext, "host-context", "", "The name of the host-kubeconfig context.") return cmd } func (o *CommandUnJoinOptions) Complete(f ctlutil.Factory) error { - var hostConfig *restclient.Config - var clusterConfig *restclient.Config - var err error - - if o.HostKubeConfig != "" { - hostConfig, err = clientcmd.BuildConfigFromFlags("", o.HostKubeConfig) - if err != nil { - return fmt.Errorf("kosmosctl unjoin complete error, generate masterConfig failed: %s", err) - } - } else { - hostConfig, err = f.ToRESTConfig() - if err != nil { - return fmt.Errorf("kosmosctl unjoin complete error, get current masterConfig failed: %s", err) - } + hostConfig, err := utils.RestConfig(o.HostKubeConfig, o.HostContext) + if err != nil { + return fmt.Errorf("kosmosctl unjoin complete error, generate host config failed: %s", err) } + var clusterConfig *restclient.Config + o.KosmosClient, err = versioned.NewForConfig(hostConfig) if err != nil { - return fmt.Errorf("kosmosctl install complete error, generate Kosmos client failed: %v", err) + return fmt.Errorf("kosmosctl unjoin complete error, generate Kosmos client failed: %v", err) } if o.KubeConfig != "" { - clusterConfig, err = clientcmd.BuildConfigFromFlags("", o.KubeConfig) + clusterConfig, err = utils.RestConfig(o.KubeConfig, o.Context) if err != nil { - return fmt.Errorf("kosmosctl unjoin complete error, generate clusterConfig failed: %s", err) + return fmt.Errorf("kosmosctl unjoin complete error, generate config failed: %s", err) } } else { var cluster *v1alpha1.Cluster diff --git a/pkg/utils/k8s.go b/pkg/utils/k8s.go index f22024a25..8c4098c9c 100644 --- a/pkg/utils/k8s.go +++ b/pkg/utils/k8s.go @@ -2,7 +2,6 @@ package utils import ( "encoding/json" - "fmt" "strings" jsonpatch "github.com/evanphx/json-patch" @@ -10,14 +9,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/metrics/pkg/client/clientset/versioned" - - kosmosversioned "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" ) +// Todo rename the filename + type ClustersNodeSelection struct { NodeSelector map[string]string `json:"nodeSelector,omitempty"` Affinity *corev1.Affinity `json:"affinity,omitempty"` @@ -47,166 +42,6 @@ func CreateMergePatch(original, new interface{}) ([]byte, error) { return patch, nil } -type Opts func(*rest.Config) - -func NewConfigFromBytes(kubeConfig []byte, opts ...Opts) (*rest.Config, error) { - var ( - config *rest.Config - err error - ) - - c, err := clientcmd.NewClientConfigFromBytes(kubeConfig) - if err != nil { - return nil, err - } - config, err = c.ClientConfig() - if err != nil { - return nil, err - } - - for _, h := range opts { - if h == nil { - continue - } - h(config) - } - - return config, nil -} - -func NewClientFromConfigPath(configPath string, opts ...Opts) (kubernetes.Interface, error) { - var ( - config *rest.Config - err error - ) - config, err = clientcmd.BuildConfigFromFlags("", configPath) - if err != nil { - return nil, fmt.Errorf("failed to build config from configpath: %v", err) - } - - for _, opt := range opts { - if opt == nil { - continue - } - opt(config) - } - - client, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, fmt.Errorf("could not create clientset: %v", err) - } - return client, nil -} - -func NewKosmosClientFromConfigPath(configPath string, opts ...Opts) (kosmosversioned.Interface, error) { - var ( - config *rest.Config - err error - ) - config, err = clientcmd.BuildConfigFromFlags("", configPath) - if err != nil { - return nil, fmt.Errorf("failed to build config from configpath: %v", err) - } - - for _, opt := range opts { - if opt == nil { - continue - } - opt(config) - } - - client, err := kosmosversioned.NewForConfig(config) - if err != nil { - return nil, fmt.Errorf("could not create clientset: %v", err) - } - return client, nil -} - -func NewClientFromBytes(kubeConfig []byte, opts ...Opts) (kubernetes.Interface, error) { - var ( - config *rest.Config - err error - ) - - clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeConfig) - if err != nil { - return nil, err - } - config, err = clientConfig.ClientConfig() - if err != nil { - return nil, err - } - - for _, opt := range opts { - if opt == nil { - continue - } - opt(config) - } - - client, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, fmt.Errorf("create client failed: %v", err) - } - return client, nil -} - -func NewKosmosClientFromBytes(kubeConfig []byte, opts ...Opts) (kosmosversioned.Interface, error) { - var ( - config *rest.Config - err error - ) - - clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeConfig) - if err != nil { - return nil, err - } - config, err = clientConfig.ClientConfig() - if err != nil { - return nil, err - } - - for _, opt := range opts { - if opt == nil { - continue - } - opt(config) - } - - client, err := kosmosversioned.NewForConfig(config) - if err != nil { - return nil, fmt.Errorf("create client failed: %v", err) - } - return client, nil -} - -func NewMetricClient(configPath string, opts ...Opts) (versioned.Interface, error) { - var ( - config *rest.Config - err error - ) - config, err = clientcmd.BuildConfigFromFlags("", configPath) - if err != nil { - config, err = rest.InClusterConfig() - if err != nil { - return nil, fmt.Errorf("could not read config file for cluster: %v", err) - } - } - - for _, opt := range opts { - if opt == nil { - continue - } - opt(config) - } - - metricClient, err := versioned.NewForConfig(config) - if err != nil { - return nil, fmt.Errorf("could not create client for root cluster: %v", err) - } - return metricClient, nil -} - func IsKosmosNode(node *corev1.Node) bool { if node == nil { return false diff --git a/pkg/utils/kube_client.go b/pkg/utils/kube_client.go new file mode 100644 index 000000000..65ca71e68 --- /dev/null +++ b/pkg/utils/kube_client.go @@ -0,0 +1,211 @@ +package utils + +import ( + "fmt" + "os" + "path/filepath" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/util/homedir" + + kosmosversioned "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" +) + +var ( + defaultKubeConfig = filepath.Join(homedir.HomeDir(), ".kube", "config") +) + +func loadKubeconfig(kubeconfigPath, context string) (*clientcmdapi.Config, error) { + if kubeconfigPath == "" { + kubeconfigPath = GetEnvString("KUBECONFIG", defaultKubeConfig) + } + + if _, err := os.Stat(kubeconfigPath); err != nil { + return nil, fmt.Errorf("kubeconfig path %s does not exist", kubeconfigPath) + } + + pathOptions := clientcmd.NewDefaultPathOptions() + loadingRules := *pathOptions.LoadingRules + loadingRules.ExplicitPath = kubeconfigPath + overrides := &clientcmd.ConfigOverrides{ + CurrentContext: context, + } + + clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&loadingRules, overrides) + rawConfig, err := clientConfig.RawConfig() + if err != nil { + return nil, err + } + + return &rawConfig, nil +} + +// RestConfig creates a rest config from the context and kubeconfig. +func RestConfig(kubeconfigPath, context string) (*rest.Config, error) { + rawConfig, err := loadKubeconfig(kubeconfigPath, context) + if err != nil { + return nil, err + } + + restConfig, err := clientcmd.NewDefaultClientConfig(*rawConfig, &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + return nil, err + } + + return restConfig, nil +} + +// RawConfig creates a raw config from the context and kubeconfig. +func RawConfig(kubeconfigPath, context string) (clientcmdapi.Config, error) { + rawConfig, err := loadKubeconfig(kubeconfigPath, context) + if err != nil { + return clientcmdapi.Config{}, err + } + + return *rawConfig, nil +} + +// GetEnvString returns the env variable,if the env is not set,return the defaultValue +func GetEnvString(key, defaultValue string) string { + v, ok := os.LookupEnv(key) + if ok { + return v + } + return defaultValue +} + +type Opts func(*rest.Config) + +func NewConfigFromBytes(kubeConfig []byte, opts ...Opts) (*rest.Config, error) { + var ( + config *rest.Config + err error + ) + + c, err := clientcmd.NewClientConfigFromBytes(kubeConfig) + if err != nil { + return nil, err + } + config, err = c.ClientConfig() + if err != nil { + return nil, err + } + + for _, h := range opts { + if h == nil { + continue + } + h(config) + } + + return config, nil +} + +func NewClientFromConfigPath(configPath string, opts ...Opts) (kubernetes.Interface, error) { + var ( + config *rest.Config + err error + ) + config, err = clientcmd.BuildConfigFromFlags("", configPath) + if err != nil { + return nil, fmt.Errorf("failed to build config from configpath: %v", err) + } + + for _, opt := range opts { + if opt == nil { + continue + } + opt(config) + } + + client, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("could not create clientset: %v", err) + } + return client, nil +} + +func NewKosmosClientFromConfigPath(configPath string, opts ...Opts) (kosmosversioned.Interface, error) { + var ( + config *rest.Config + err error + ) + config, err = clientcmd.BuildConfigFromFlags("", configPath) + if err != nil { + return nil, fmt.Errorf("failed to build config from configpath: %v", err) + } + + for _, opt := range opts { + if opt == nil { + continue + } + opt(config) + } + + client, err := kosmosversioned.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("could not create clientset: %v", err) + } + return client, nil +} + +func NewClientFromBytes(kubeConfig []byte, opts ...Opts) (kubernetes.Interface, error) { + var ( + config *rest.Config + err error + ) + + clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeConfig) + if err != nil { + return nil, err + } + config, err = clientConfig.ClientConfig() + if err != nil { + return nil, err + } + + for _, opt := range opts { + if opt == nil { + continue + } + opt(config) + } + + client, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("create client failed: %v", err) + } + return client, nil +} + +func NewKosmosClientFromBytes(kubeConfig []byte, opts ...Opts) (kosmosversioned.Interface, error) { + var ( + config *rest.Config + err error + ) + + clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeConfig) + if err != nil { + return nil, err + } + config, err = clientConfig.ClientConfig() + if err != nil { + return nil, err + } + + for _, opt := range opts { + if opt == nil { + continue + } + opt(config) + } + + client, err := kosmosversioned.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("create client failed: %v", err) + } + return client, nil +} diff --git a/pkg/utils/node_client.go b/pkg/utils/node_client.go deleted file mode 100644 index a6dd7d65c..000000000 --- a/pkg/utils/node_client.go +++ /dev/null @@ -1,85 +0,0 @@ -package utils - -import ( - "context" - - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - - kosmosv1alpha1 "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" - kosmosversioned "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" -) - -type ClusterKubeClient struct { - KubeClient kubernetes.Interface - ClusterName string -} - -type ClusterKosmosClient struct { - KosmosClient kosmosversioned.Interface - ClusterName string -} - -type ClusterDynamicClient struct { - DynamicClient dynamic.Interface - ClusterName string -} - -// NewClusterKubeClient create a kube client for a member cluster -func NewClusterKubeClient(config *rest.Config, ClusterName string) (*ClusterKubeClient, error) { - kubeClient, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, err - } - - return &ClusterKubeClient{ - KubeClient: kubeClient, - ClusterName: ClusterName, - }, nil -} - -// NewClusterKosmosClient create a dynamic client for a member cluster -func NewClusterKosmosClient(config *rest.Config, ClusterName string) (*ClusterKosmosClient, error) { - kosmosClient, err := kosmosversioned.NewForConfig(config) - if err != nil { - return nil, err - } - - return &ClusterKosmosClient{ - KosmosClient: kosmosClient, - ClusterName: ClusterName, - }, nil -} - -// NewClusterDynamicClient create a kosmos crd client for a member cluster -func NewClusterDynamicClient(config *rest.Config, ClusterName string) (*ClusterDynamicClient, error) { - dynamicClient, err := dynamic.NewForConfig(config) - if err != nil { - return nil, err - } - - return &ClusterDynamicClient{ - DynamicClient: dynamicClient, - ClusterName: ClusterName, - }, nil -} - -func BuildConfig(cluster *kosmosv1alpha1.Cluster, opts Opts) (*rest.Config, error) { - config, err := NewConfigFromBytes(cluster.Spec.Kubeconfig, opts) - if err != nil { - return nil, err - } - return config, nil -} - -// GetCluster returns the member cluster -func GetCluster(hostClient client.Client, clusterName string) (*kosmosv1alpha1.Cluster, error) { - cluster := &kosmosv1alpha1.Cluster{} - if err := hostClient.Get(context.TODO(), types.NamespacedName{Name: clusterName}, cluster); err != nil { - return nil, err - } - return cluster, nil -} diff --git a/test/e2e/deploy/cr/cluster-cr.yaml b/test/e2e/deploy/cr/cluster-cr.yaml index e723c16ae..fb29e252a 100644 --- a/test/e2e/deploy/cr/cluster-cr.yaml +++ b/test/e2e/deploy/cr/cluster-cr.yaml @@ -79,7 +79,6 @@ spec: operator: NotIn values: - cluster-host-control-plane - - cluster-host-worker podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: diff --git a/test/e2e/deploy/nginx/nginx.yaml b/test/e2e/deploy/nginx/nginx.yaml index 5c3c8a1ac..c1c13b6be 100644 --- a/test/e2e/deploy/nginx/nginx.yaml +++ b/test/e2e/deploy/nginx/nginx.yaml @@ -32,7 +32,6 @@ spec: operator: NotIn values: - cluster-host-control-plane - - cluster-host-worker podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: diff --git a/test/e2e/leaf_node_test.go b/test/e2e/leaf_node_test.go index 2e6c5c12b..7a9c2403a 100644 --- a/test/e2e/leaf_node_test.go +++ b/test/e2e/leaf_node_test.go @@ -62,8 +62,9 @@ var _ = ginkgo.Describe("Test leaf node mode -- one2cluster, one2node, one2party if i < 2 { nodeLabels := node.Labels if nodeLabels == nil { - nodeLabels = make(map[string]string, 0) + nodeLabels = make(map[string]string) } + nodeLabels["test-leaf-party-mode"] = "yes" node.SetLabels(nodeLabels) node.ResourceVersion = "" @@ -175,6 +176,7 @@ var _ = ginkgo.Describe("Test leaf node mode -- one2cluster, one2node, one2party err = framework.CreateClusters(hostClusterLinkClient, one2Node) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) if len(memberNodeNames) > 0 { + // #nosec G602 framework.WaitNodePresentOnCluster(hostKubeClient, memberNodeNames[0]) } }) @@ -227,6 +229,7 @@ var _ = ginkgo.Describe("Test leaf node mode -- one2cluster, one2node, one2party gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) if len(partyNodeNames) > 0 { + // #nosec G602 framework.WaitNodePresentOnCluster(hostKubeClient, partyNodeNames[0]) } })