feat: make kosmosctl support --context

Signed-off-by: duanmengkk <[email protected]>

hack/cluster.sh

@@ -17,6 +17,7 @@ CERT=$(cat ${ROOT}/pkg/cert/crt.pem | base64 -w 0)
 KEY=$(cat ${ROOT}/pkg/cert/key.pem | base64 -w 0)
 
 CN_ZONE=${CN_ZONE:-false}
+source "$(dirname "${BASH_SOURCE[0]}")/util.sh"
 
 if [ $REUSE == true ]; then
     echo "!!!!!!!!!!!Warning: Setting REUSE to true will not delete existing clusters.!!!!!!!!!!!"
@@ -147,8 +148,6 @@ function create_cluster() {
       "kubectl get nodes | awk 'NR>1 {if (\$2 != \"Ready\") exit 1; }' && [ \$(kubectl get nodes --no-headers | wc -l) -eq ${N} ]" \
       300
     echo "all node ready"
-
-    kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/crds/mcs
 }
 
 function join_cluster() {
@@ -185,11 +184,46 @@ EOF
   kubectl --context="kind-${member_cluster}" apply -f "$ROOT"/deploy/clusterlink-datapanel-rbac.yml
 }
 
+function join_cluster_by_ctl(){
+  local host_cluster=$1
+  local member_cluster=$2
+  HOST_CLUSTER_DIR="${ROOT}/environments/${host_cluster}"
+  MEMBER_CLUSTER_DIR="${ROOT}/environments/${member_cluster}"
+  kosmosctl join cluster --name $member_cluster --host-kubeconfig $HOST_CLUSTER_DIR/kubeconfig --kubeconfig $MEMBER_CLUSTER_DIR/kubeconfig --enable-all --version latest
+}
+
+
+function deploy_cluster_by_ctl(){
+   local -r clustername=$1
+   CLUSTER_DIR="${ROOT}/environments/${clustername}"
+   load_cluster_images "$clustername"
+   kosmosctl install --version latest --kubeconfig $CLUSTER_DIR/kubeconfig
+
+   # deploy kosmos-scheduler for e2e test case of mysql-operator
+   sed -e "s|__VERSION__|$VERSION|g" -e "w ${ROOT}/environments/kosmos-scheduler.yml" "$ROOT"/deploy/scheduler/deployment.yaml
+   kubectl --context="kind-${clustername}" apply -f "${ROOT}/environments/kosmos-scheduler.yml"
+   kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/scheduler/rbac.yaml
+
+   util::wait_for_condition "kosmos scheduler are ready" \
+     "kubectl --context="kind-${clustername}" -n kosmos-system get deploy kosmos-scheduler -o jsonpath='{.status.replicas}{\" \"}{.status.readyReplicas}{\"\n\"}' | awk '{if (\$1 == \$2 && \$1 > 0) exit 0; else exit 1}'" \
+     300
+   echo "cluster $clustername deploy kosmos-scheduler success"
+
+   docker exec ${clustername}-control-plane /bin/sh -c "mv /etc/kubernetes/manifests/kube-scheduler.yaml /etc/kubernetes"
+
+   # add the args for e2e test case of mysql-operator
+   kubectl --context="kind-${clustername}" -n kosmos-system patch deployment clustertree-cluster-manager --type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--auto-mcs-prefix=kosmos-e2e"}]'
+
+   util::wait_for_condition "kosmos clustertree are ready" \
+     "kubectl --context="kind-${clustername}" -n kosmos-system get deploy clustertree-cluster-manager -o jsonpath='{.status.replicas}{\" \"}{.status.readyReplicas}{\"\n\"}' | awk '{if (\$1 == \$2 && \$1 > 0) exit 0; else exit 1}'" \
+     300
+}
+
 function deploy_cluster() {
    local -r clustername=$1
    kubectl config use-context "kind-${clustername}"
    load_cluster_images "$clustername"
-
+   
    kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/clusterlink-namespace.yml
    kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/kosmos-rbac.yml
    kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/crds

hack/prepare-e2e.sh

@@ -4,8 +4,6 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-ENABLED_MYSQL_E2E=${MYSQL_E2E:-true}
-
 KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}
 export KUBECONFIG=$KUBECONFIG_PATH/"config"
 
@@ -27,16 +25,20 @@ source "$(dirname "${BASH_SOURCE[0]}")/install_kind_kubectl.sh"
 source "$(dirname "${BASH_SOURCE[0]}")/cluster.sh"
 make images GOOS="linux" --directory="${ROOT}"
 
+make kosmosctl
+os=$(go env GOOS)
+arch=$(go env GOARCH)
+export PATH=$PATH:"$ROOT"/_output/bin/"$os"/"$arch"
+
 #cluster cluster
 create_cluster $HOST_CLUSTER_NAME $HOST_CLUSTER_POD_CIDR $HOST_CLUSTER_SERVICE_CIDR
 create_cluster $MEMBER1_CLUSTER_NAME $MEMBER1_CLUSTER_POD_CIDR $MEMBER1_CLUSTER_SERVICE_CIDR false
 create_cluster $MEMBER2_CLUSTER_NAME $MEMBER2_CLUSTER_POD_CIDR $MEMBER2_CLUSTER_SERVICE_CIDR fasle
 #deploy cluster
-deploy_cluster $HOST_CLUSTER_NAME
+deploy_cluster_by_ctl $HOST_CLUSTER_NAME
 load_cluster_images $MEMBER1_CLUSTER_NAME
 load_cluster_images $MEMBER2_CLUSTER_NAME
 
 #join cluster
-join_cluster $HOST_CLUSTER_NAME $HOST_CLUSTER_NAME
-join_cluster $HOST_CLUSTER_NAME $MEMBER1_CLUSTER_NAME
-join_cluster $HOST_CLUSTER_NAME $MEMBER2_CLUSTER_NAME
+join_cluster_by_ctl $HOST_CLUSTER_NAME $MEMBER1_CLUSTER_NAME
+join_cluster_by_ctl $HOST_CLUSTER_NAME $MEMBER2_CLUSTER_NAME

pkg/kosmosctl/floater/analysis.go

@@ -35,6 +35,7 @@ type CommandAnalysisOptions struct {
 	ImageRepository string
 	Version         string
 	KubeConfig      string
+	Context         string
 
 	Port        string
 	PodWaitTime int
@@ -78,6 +79,7 @@ func NewCmdAnalysis(f ctlutil.Factory) *cobra.Command {
 	flags.StringVarP(&o.ImageRepository, "image-repository", "r", utils.DefaultImageRepository, "Image repository.")
 	flags.StringVar(&o.Name, "name", "", "Specify the name of the resource to analysis.")
 	flags.StringVar(&o.KubeConfig, "kubeconfig", "", "Absolute path to the cluster kubeconfig file.")
+	flags.StringVar(&o.Context, "context", "", "The name of the kubeconfig context.")
 	flags.StringVar(&o.Port, "port", utils.DefaultPort, "Port used by floater.")
 	flags.IntVarP(&o.PodWaitTime, "pod-wait-time", "w", utils.DefaultWaitTime, "Time for wait pod(floater) launch.")
 	flags.BoolVar(&o.GenGraph, "gen-graph", false, "Configure generate network analysis graph.")
@@ -97,7 +99,7 @@ func (o *CommandAnalysisOptions) Complete(f ctlutil.Factory) error {
 	}
 
 	af := NewAnalysisFloater(o)
-	if err = af.completeFromKubeConfigPath(o.KubeConfig); err != nil {
+	if err = af.completeFromKubeConfigPath(o.KubeConfig, o.Context); err != nil {
 		return err
 	}
 	o.Floater = af

pkg/kosmosctl/floater/check.go

@@ -40,8 +40,12 @@ type CommandCheckOptions struct {
 	HostNetwork bool
 
 	KubeConfig    string
+	Context       string
 	SrcKubeConfig string
+	SrcContext    string
+
 	DstKubeConfig string
+	DstContext    string
 
 	SrcFloater *Floater
 	DstFloater *Floater
@@ -86,8 +90,11 @@ func NewCmdCheck() *cobra.Command {
 	flags.StringVarP(&o.ImageRepository, "image-repository", "r", utils.DefaultImageRepository, "Image repository.")
 	flags.StringVarP(&o.DstImageRepository, "dst-image-repository", "", "", "Destination cluster image repository.")
 	flags.StringVar(&o.KubeConfig, "kubeconfig", "", "Absolute path to the host kubeconfig file.")
+	flags.StringVar(&o.Context, "context", "", "The name of the kubeconfig context.")
 	flags.StringVar(&o.SrcKubeConfig, "src-kubeconfig", "", "Absolute path to the source cluster kubeconfig file.")
+	flags.StringVar(&o.SrcContext, "src-context", "", "The name of the src-kubeconfig context.")
 	flags.StringVar(&o.DstKubeConfig, "dst-kubeconfig", "", "Absolute path to the destination cluster kubeconfig file.")
+	flags.StringVar(&o.DstContext, "dst-context", "", "The name of the dst-kubeconfig context.")
 	flags.BoolVar(&o.HostNetwork, "host-network", false, "Configure HostNetwork.")
 	flags.StringVar(&o.Port, "port", "8889", "Port used by floater.")
 	flags.IntVarP(&o.PodWaitTime, "pod-wait-time", "w", 30, "Time for wait pod(floater) launch.")
@@ -102,14 +109,14 @@ func (o *CommandCheckOptions) Complete() error {
 	}
 
 	srcFloater := NewCheckFloater(o, false)
-	if err := srcFloater.completeFromKubeConfigPath(o.SrcKubeConfig); err != nil {
+	if err := srcFloater.completeFromKubeConfigPath(o.SrcKubeConfig, o.SrcContext); err != nil {
 		return err
 	}
 	o.SrcFloater = srcFloater
 
 	if o.DstKubeConfig != "" {
 		dstFloater := NewCheckFloater(o, true)
-		if err := dstFloater.completeFromKubeConfigPath(o.DstKubeConfig); err != nil {
+		if err := dstFloater.completeFromKubeConfigPath(o.DstKubeConfig, o.DstContext); err != nil {
 			return err
 		}
 		o.DstFloater = dstFloater

pkg/kosmosctl/floater/floater.go

@@ -14,7 +14,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/remotecommand"
 	"k8s.io/klog/v2"
 
@@ -102,13 +101,13 @@ func NewAnalysisFloater(o *CommandAnalysisOptions) *Floater {
 	return floater
 }
 
-func (f *Floater) completeFromKubeConfigPath(kubeConfigPath string) error {
-	config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
+func (f *Floater) completeFromKubeConfigPath(kubeConfigPath, context string) error {
+	config, err := utils.RestConfig(kubeConfigPath, context)
 	if err != nil {
 		return fmt.Errorf("kosmosctl docter complete error, generate floater config failed: %v", err)
 	}
-	f.Config = config
 
+	f.Config = config
 	f.Client, err = kubernetes.NewForConfig(f.Config)
 	if err != nil {
 		return fmt.Errorf("kosmosctl docter complete error, generate floater client failed: %v", err)

pkg/kosmosctl/install/install.go

@@ -3,8 +3,6 @@ package install
 import (
 	"context"
 	"fmt"
-	"os"
-	"path/filepath"
 
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
@@ -13,9 +11,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/util/homedir"
 	"k8s.io/klog/v2"
 	ctlutil "k8s.io/kubectl/pkg/cmd/util"
 	"k8s.io/kubectl/pkg/util/i18n"
@@ -53,6 +49,7 @@ type CommandInstallOptions struct {
 	Version              string
 	Module               string
 	HostKubeConfig       string
+	HostContext          string
 	HostKubeConfigStream []byte
 	WaitTime             int
 
@@ -71,7 +68,7 @@ type CommandInstallOptions struct {
 }
 
 // NewCmdInstall Install the Kosmos control plane in a Kubernetes cluster.
-func NewCmdInstall(f ctlutil.Factory) *cobra.Command {
+func NewCmdInstall() *cobra.Command {
 	o := &CommandInstallOptions{}
 
 	cmd := &cobra.Command{
@@ -82,7 +79,7 @@ func NewCmdInstall(f ctlutil.Factory) *cobra.Command {
 		SilenceUsage:          true,
 		DisableFlagsInUseLine: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctlutil.CheckErr(o.Complete(f))
+			ctlutil.CheckErr(o.Complete())
 			ctlutil.CheckErr(o.Validate())
 			ctlutil.CheckErr(o.Run())
 			return nil
@@ -92,8 +89,10 @@ func NewCmdInstall(f ctlutil.Factory) *cobra.Command {
 	flags := cmd.Flags()
 	flags.StringVarP(&o.Namespace, "namespace", "n", utils.DefaultNamespace, "Kosmos namespace.")
 	flags.StringVarP(&o.ImageRegistry, "private-image-registry", "", utils.DefaultImageRepository, "Private image registry where pull images from. If set, all required images will be downloaded from it, it would be useful in offline installation scenarios.  In addition, you still can use --kube-image-registry to specify the registry for Kubernetes's images.")
+	flags.StringVar(&o.Version, "version", "", "image version for pull images")
 	flags.StringVarP(&o.Module, "module", "m", utils.All, "Kosmos specify the module to install.")
 	flags.StringVar(&o.HostKubeConfig, "kubeconfig", "", "Absolute path to the special kubeconfig file.")
+	flags.StringVar(&o.HostContext, "context", "", "The name of the kubeconfig context.")
 	flags.StringVar(&o.CNI, "cni", "", "The cluster is configured using cni and currently supports calico and flannel.")
 	flags.StringVar(&o.DefaultNICName, "default-nic", "", "Set default network interface card.")
 	flags.StringVar(&o.NetworkType, "network-type", utils.NetworkTypeGateway, "Set the cluster network connection mode, which supports gateway and p2p modes, gateway is used by default.")
@@ -107,35 +106,32 @@ func NewCmdInstall(f ctlutil.Factory) *cobra.Command {
 	return cmd
 }
 
-func (o *CommandInstallOptions) Complete(f ctlutil.Factory) error {
-	var config *rest.Config
-	var err error
+func (o *CommandInstallOptions) Complete() error {
+	config, err := utils.RestConfig(o.HostKubeConfig, o.HostContext)
+	if err != nil {
+		return fmt.Errorf("kosmosctl install complete error, generate host config failed: %s", err)
+	}
 
-	if len(o.HostKubeConfig) > 0 {
-		config, err = clientcmd.BuildConfigFromFlags("", o.HostKubeConfig)
-		if err != nil {
-			return fmt.Errorf("kosmosctl install complete error, generate host config failed: %s", err)
-		}
-		o.HostKubeConfigStream, err = os.ReadFile(o.HostKubeConfig)
-		if err != nil {
-			return fmt.Errorf("kosmosctl install complete error, read host config failed: %s", err)
-		}
-	} else {
-		config, err = f.ToRESTConfig()
-		if err != nil {
-			return fmt.Errorf("kosmosctl install complete error, generate rest config failed: %v", err)
-		}
-		o.HostKubeConfigStream, err = os.ReadFile(filepath.Join(homedir.HomeDir(), ".kube", "config"))
-		if err != nil {
-			return fmt.Errorf("kosmosctl install complete error, read host config failed: %s", err)
-		}
+	rawConfig, err := utils.RawConfig(o.HostKubeConfig, o.HostContext)
+	if err != nil {
+		return fmt.Errorf("kosmosctl install complete error, generate host config failed: %s", err)
+	}
+
+	streams, err := clientcmd.Write(rawConfig)
+	if err != nil {
+		return fmt.Errorf("kosmosctl install complete error, generate host config streams failed: %s", err)
 	}
 
+	o.HostKubeConfigStream = streams
 	o.KosmosClient, err = versioned.NewForConfig(config)
 	if err != nil {
 		return fmt.Errorf("kosmosctl install complete error, generate Kosmos client failed: %v", err)
 	}
 
+	if o.Version == "" {
+		o.Version = fmt.Sprintf("v%s", version.GetReleaseVersion().PatchRelease())
+	}
+
 	o.K8sClient, err = kubernetes.NewForConfig(config)
 	if err != nil {
 		return fmt.Errorf("kosmosctl install complete error, generate K8s basic client failed: %v", err)
@@ -292,7 +288,7 @@ func (o *CommandInstallOptions) runClusterlink() error {
 	networkManagerDeploy, err := util.GenerateDeployment(manifest.ClusterlinkNetworkManagerDeployment, manifest.DeploymentReplace{
 		Namespace:       o.Namespace,
 		ImageRepository: o.ImageRegistry,
-		Version:         version.GetReleaseVersion().PatchRelease(),
+		Version:         o.Version,
 	})
 	if err != nil {
 		return err
@@ -312,7 +308,7 @@ func (o *CommandInstallOptions) runClusterlink() error {
 
 	operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{
 		Namespace:       o.Namespace,
-		Version:         version.GetReleaseVersion().PatchRelease(),
+		Version:         o.Version,
 		UseProxy:        o.UseProxy,
 		ImageRepository: o.ImageRegistry,
 	})
@@ -443,7 +439,7 @@ func (o *CommandInstallOptions) runClustertree() error {
 	clustertreeDeploy, err := util.GenerateDeployment(manifest.ClusterTreeClusterManagerDeployment, manifest.DeploymentReplace{
 		Namespace:       o.Namespace,
 		ImageRepository: o.ImageRegistry,
-		Version:         version.GetReleaseVersion().PatchRelease(),
+		Version:         o.Version,
 	})
 	if err != nil {
 		return err
@@ -463,7 +459,7 @@ func (o *CommandInstallOptions) runClustertree() error {
 
 	operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{
 		Namespace:       o.Namespace,
-		Version:         version.GetReleaseVersion().PatchRelease(),
+		Version:         o.Version,
 		UseProxy:        o.UseProxy,
 		ImageRepository: o.ImageRegistry,
 	})
@@ -489,7 +485,7 @@ func (o *CommandInstallOptions) createOperator() error {
 	klog.Info("Start creating Kosmos-Operator...")
 	operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{
 		Namespace:       o.Namespace,
-		Version:         version.GetReleaseVersion().PatchRelease(),
+		Version:         o.Version,
 		UseProxy:        o.UseProxy,
 		ImageRepository: o.ImageRegistry,
 	})