support --configfile for kubenest

cmd/kubenest/operator/app/config/config.go

@@ -0,0 +1,19 @@
+package config
+
+import (
+	clientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	componentbaseconfig "k8s.io/component-base/config"
+
+	"github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1"
+)
+
+// Config has all the configurations for kubenest.
+type Config struct {
+	KubeNestOptions  v1alpha1.KubeNestConfiguration
+	Client           clientset.Interface
+	RestConfig       *restclient.Config
+	KubeconfigStream []byte
+	// LeaderElection is optional.
+	LeaderElection componentbaseconfig.LeaderElectionConfiguration
+}

cmd/kubenest/operator/app/operator.go

@@ -3,17 +3,24 @@ package app
 import (
 	"context"
 	"fmt"
+	"os"
 
 	"github.com/spf13/cobra"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/client-go/kubernetes"
+	clientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/klog/v2"
+	kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
+	"github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/config"
 	"github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options"
+	"github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1"
 	"github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned"
 	"github.com/kosmos.io/kosmos/pkg/kubenest/constants"
 	"github.com/kosmos.io/kosmos/pkg/kubenest/controller"
@@ -32,10 +39,7 @@ func NewVirtualClusterOperatorCommand(ctx context.Context) *cobra.Command {
 		Use:  "virtual-cluster-operator",
 		Long: `create virtual kubernetes control plane with VirtualCluster`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			if errs := opts.Validate(); len(errs) != 0 {
-				return errs.ToAggregate()
-			}
-			if err := run(ctx, opts); err != nil {
+			if err := runCommand(ctx, opts); err != nil {
 				return err
 			}
 			return nil
@@ -56,6 +60,129 @@ func NewVirtualClusterOperatorCommand(ctx context.Context) *cobra.Command {
 	return cmd
 }
 
+func runCommand(ctx context.Context, opts *options.Options) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	kc, err := SetupConfig(opts)
+	if err != nil {
+		return err
+	}
+	return run(ctx, kc)
+}
+
+func SetupConfig(opts *options.Options) (*config.Config, error) {
+	c := &config.Config{}
+
+	var koc v1alpha1.KubeNestConfiguration
+	if len(opts.ConfigFile) != 0 {
+		ko, err := loadConfig(opts.ConfigFile)
+		if err != nil {
+			return nil, err
+		}
+		koc = *ko
+	} else {
+		ko := &v1alpha1.KubeNestConfiguration{}
+		ko.KubeNestType = v1alpha1.KubeInKube
+		ko.KosmosKubeConfig.AllowNodeOwnbyMulticluster = false
+		ko.KubeInKubeConfig.ForceDestroy = opts.DeprecatedOptions.KubeInKubeConfig.ForceDestroy
+		ko.KubeInKubeConfig.ETCDUnitSize = opts.DeprecatedOptions.KubeInKubeConfig.ETCDUnitSize
+		ko.KubeInKubeConfig.ETCDStorageClass = opts.DeprecatedOptions.KubeInKubeConfig.ETCDStorageClass
+		ko.KubeInKubeConfig.AdmissionPlugins = opts.DeprecatedOptions.KubeInKubeConfig.AdmissionPlugins
+		ko.KubeInKubeConfig.AnpMode = opts.DeprecatedOptions.KubeInKubeConfig.AnpMode
+		ko.KubeInKubeConfig.ApiServerReplicas = opts.DeprecatedOptions.KubeInKubeConfig.ApiServerReplicas
+		ko.KubeInKubeConfig.ClusterCIDR = opts.DeprecatedOptions.KubeInKubeConfig.ClusterCIDR
+
+		koc = *ko
+	}
+
+	fillInForDefault(c, koc)
+	printKubeNestConfiguration(koc)
+
+	kubeconfigStream, err := os.ReadFile(opts.KubernetesOptions.KubeConfig)
+	if err != nil {
+		return nil, fmt.Errorf("read kubeconfig file failed: %v", err)
+	}
+
+	// Prepare kube config.
+	kubeConfig, err := createKubeConfig(opts)
+	if err != nil {
+		return nil, err
+	}
+
+	// Prepare kube clients.
+	client, err := createClients(kubeConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	c.KubeconfigStream = kubeconfigStream
+	c.RestConfig = kubeConfig
+	c.Client = client
+	c.LeaderElection = opts.LeaderElection
+	c.KubeNestOptions = koc
+
+	return c, nil
+}
+
+// TODO
+func printKubeNestConfiguration(koc v1alpha1.KubeNestConfiguration) {
+
+}
+
+// TODO
+func fillInForDefault(c *config.Config, koc v1alpha1.KubeNestConfiguration) {
+
+}
+
+func loadConfig(file string) (*v1alpha1.KubeNestConfiguration, error) {
+	data, err := os.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	// The UniversalDecoder runs defaulting and returns the internal type by default.
+	obj, gvk, err := kubeschedulerscheme.Codecs.UniversalDecoder().Decode(data, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	if cfgObj, ok := obj.(*v1alpha1.KubeNestConfiguration); ok {
+		return cfgObj, nil
+	}
+	return nil, fmt.Errorf("couldn't decode as KubeNestConfiguration, got %s: ", gvk)
+}
+
+// createClients creates a kube client and an event client from the given kubeConfig
+func createClients(kubeConfig *restclient.Config) (clientset.Interface, error) {
+	client, err := clientset.NewForConfig(kubeConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return client, nil
+}
+
+// createKubeConfig creates a kubeConfig from the given config and masterOverride.
+func createKubeConfig(opts *options.Options) (*restclient.Config, error) {
+	if len(opts.KubernetesOptions.KubeConfig) == 0 && len(opts.KubernetesOptions.Master) == 0 {
+		klog.Warning("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.")
+	}
+
+	// This creates a client, first loading any specified kubeconfig
+	// file, and then overriding the Master flag, if non-empty.
+	kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: opts.KubernetesOptions.KubeConfig},
+		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: opts.KubernetesOptions.Master}}).ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	kubeConfig.DisableCompression = true
+	kubeConfig.QPS = opts.KubernetesOptions.QPS
+	kubeConfig.Burst = opts.KubernetesOptions.Burst
+
+	return kubeConfig, nil
+}
+
 func startEndPointsControllers(mgr manager.Manager) error {
 	coreEndPointsController := endpointscontroller.CoreDNSController{
 		Client:        mgr.GetClient(),
@@ -87,36 +214,30 @@ func startEndPointsControllers(mgr manager.Manager) error {
 	return nil
 }
 
-func run(ctx context.Context, opts *options.Options) error {
-	config, err := clientcmd.BuildConfigFromFlags(opts.KubernetesOptions.Master, opts.KubernetesOptions.KubeConfig)
-	if err != nil {
-		panic(err)
-	}
-	config.QPS, config.Burst = opts.KubernetesOptions.QPS, opts.KubernetesOptions.Burst
-
+func run(ctx context.Context, config *config.Config) error {
 	newscheme := scheme.NewSchema()
-	err = apiextensionsv1.AddToScheme(newscheme)
+	err := apiextensionsv1.AddToScheme(newscheme)
 	if err != nil {
 		panic(err)
 	}
 
-	mgr, err := controllerruntime.NewManager(config, controllerruntime.Options{
+	mgr, err := controllerruntime.NewManager(config.RestConfig, controllerruntime.Options{
 		Logger:                  klog.Background(),
 		Scheme:                  newscheme,
-		LeaderElection:          opts.LeaderElection.LeaderElect,
-		LeaderElectionID:        opts.LeaderElection.ResourceName,
-		LeaderElectionNamespace: opts.LeaderElection.ResourceNamespace,
+		LeaderElection:          config.LeaderElection.LeaderElect,
+		LeaderElectionID:        config.LeaderElection.ResourceName,
+		LeaderElectionNamespace: config.LeaderElection.ResourceNamespace,
 	})
 	if err != nil {
 		return fmt.Errorf("failed to build controller manager: %v", err)
 	}
 
-	hostKubeClient, err := kubernetes.NewForConfig(config)
+	hostKubeClient, err := kubernetes.NewForConfig(config.RestConfig)
 	if err != nil {
 		return fmt.Errorf("could not create clientset: %v", err)
 	}
 
-	kosmosClient, err := versioned.NewForConfig(config)
+	kosmosClient, err := versioned.NewForConfig(config.RestConfig)
 	if err != nil {
 		return fmt.Errorf("could not create clientset: %v", err)
 	}
@@ -127,7 +248,7 @@ func run(ctx context.Context, opts *options.Options) error {
 		EventRecorder:   mgr.GetEventRecorderFor(constants.InitControllerName),
 		RootClientSet:   hostKubeClient,
 		KosmosClient:    kosmosClient,
-		KubeNestOptions: &opts.KubeNestOptions,
+		KubeNestOptions: &config.KubeNestOptions,
 	}
 	if err = VirtualClusterInitController.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error starting %s: %v", constants.InitControllerName, err)
@@ -153,19 +274,20 @@ func run(ctx context.Context, opts *options.Options) error {
 		hostKubeClient,
 		mgr.GetEventRecorderFor(constants.NodeControllerName),
 		kosmosClient,
-		&opts.KubeNestOptions,
+		&config.KubeNestOptions,
 	)
 
 	if err = VirtualClusterNodeController.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error starting %s: %v", constants.NodeControllerName, err)
 	}
 
-	if opts.KosmosJoinController {
+	if config.KubeNestOptions.KubeNestType == v1alpha1.KosmosKube {
 		KosmosJoinController := kosmos.KosmosJoinController{
 			Client:                     mgr.GetClient(),
 			EventRecorder:              mgr.GetEventRecorderFor(constants.KosmosJoinControllerName),
-			KubeconfigPath:             opts.KubernetesOptions.KubeConfig,
-			AllowNodeOwnbyMulticluster: opts.AllowNodeOwnbyMulticluster,
+			KubeConfig:                 config.RestConfig,
+			KubeconfigStream:           config.KubeconfigStream,
+			AllowNodeOwnbyMulticluster: config.KubeNestOptions.KosmosKubeConfig.AllowNodeOwnbyMulticluster,
 		}
 		if err = KosmosJoinController.SetupWithManager(mgr); err != nil {
 			return fmt.Errorf("error starting %s: %v", constants.KosmosJoinControllerName, err)

cmd/kubenest/operator/app/options/options.go

@@ -5,15 +5,19 @@ import (
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	componentbaseconfig "k8s.io/component-base/config"
 
+	"github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1"
 	"github.com/kosmos.io/kosmos/pkg/utils"
 )
 
 type Options struct {
 	LeaderElection             componentbaseconfig.LeaderElectionConfiguration
 	KubernetesOptions          KubernetesOptions
-	KubeNestOptions            KubeNestOptions
+	DeprecatedOptions          v1alpha1.KubeNestConfiguration
 	AllowNodeOwnbyMulticluster bool
 	KosmosJoinController       bool
+
+	// ConfigFile is the location of the kubenest's configuration file.
+	ConfigFile string
 }
 
 type KubernetesOptions struct {
@@ -58,11 +62,12 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&o.KubernetesOptions.Master, "master", "", "Used to generate kubeconfig for downloading, if not specified, will use host in kubeconfig.")
 	flags.BoolVar(&o.AllowNodeOwnbyMulticluster, "multiowner", false, "Allow node own by multicluster or not.")
 	flags.BoolVar(&o.KosmosJoinController, "kosmos-join-controller", false, "Turn on or off kosmos-join-controller.")
-	flags.BoolVar(&o.KubeNestOptions.ForceDestroy, "kube-nest-force-destroy", false, "Force destroy the node.If it set true.If set to true, Kubernetes will not evict the existing nodes on the node when joining nodes to the tenant's control plane, but will instead force destroy.")
-	flags.StringVar(&o.KubeNestOptions.AnpMode, "kube-nest-anp-mode", "tcp", "kube-apiserver network proxy mode, must be set to tcp or uds. uds mode the replicas for apiserver should be one, and tcp for multi apiserver replicas.")
-	flags.BoolVar(&o.KubeNestOptions.AdmissionPlugins, "kube-nest-admission-plugins", false, "kube-apiserver network disable-admission-plugins, false for - --disable-admission-plugins=License, true for remove the --disable-admission-plugins=License flag .")
-	flags.IntVar(&o.KubeNestOptions.ApiServerReplicas, "kube-nest-apiserver-replicas", 1, "virtual-cluster kube-apiserver replicas. default is 2.")
-	flags.StringVar(&o.KubeNestOptions.ClusterCIDR, "cluster-cidr", "10.244.0.0/16", "Used to set the cluster-cidr of kube-controller-manager and kube-proxy (configmap)")
-	flags.StringVar(&o.KubeNestOptions.ETCDStorageClass, "etcd-storage-class", "openebs-hostpath", "Used to set the etcd storage class.")
-	flags.StringVar(&o.KubeNestOptions.ETCDUnitSize, "etcd-unit-size", "1Gi", "Used to set the etcd unit size, each node is allocated storage of etcd-unit-size.")
+	flags.BoolVar(&o.DeprecatedOptions.KubeInKubeConfig.ForceDestroy, "kube-nest-force-destroy", false, "Force destroy the node.If it set true.If set to true, Kubernetes will not evict the existing nodes on the node when joining nodes to the tenant's control plane, but will instead force destroy.")
+	flags.StringVar(&o.DeprecatedOptions.KubeInKubeConfig.AnpMode, "kube-nest-anp-mode", "tcp", "kube-apiserver network proxy mode, must be set to tcp or uds. uds mode the replicas for apiserver should be one, and tcp for multi apiserver replicas.")
+	flags.BoolVar(&o.DeprecatedOptions.KubeInKubeConfig.AdmissionPlugins, "kube-nest-admission-plugins", false, "kube-apiserver network disable-admission-plugins, false for - --disable-admission-plugins=License, true for remove the --disable-admission-plugins=License flag .")
+	flags.IntVar(&o.DeprecatedOptions.KubeInKubeConfig.ApiServerReplicas, "kube-nest-apiserver-replicas", 1, "virtual-cluster kube-apiserver replicas. default is 2.")
+	flags.StringVar(&o.DeprecatedOptions.KubeInKubeConfig.ClusterCIDR, "cluster-cidr", "10.244.0.0/16", "Used to set the cluster-cidr of kube-controller-manager and kube-proxy (configmap)")
+	flags.StringVar(&o.DeprecatedOptions.KubeInKubeConfig.ETCDStorageClass, "etcd-storage-class", "openebs-hostpath", "Used to set the etcd storage class.")
+	flags.StringVar(&o.DeprecatedOptions.KubeInKubeConfig.ETCDUnitSize, "etcd-unit-size", "1Gi", "Used to set the etcd unit size, each node is allocated storage of etcd-unit-size.")
+	flags.StringVar(&o.ConfigFile, "config", "", "The path to the configuration file.")
 }

pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go

@@ -0,0 +1,65 @@
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type KubeNestType string
+
+const (
+	KubeInKube KubeNestType = "Kube in kube"
+	KosmosKube KubeNestType = "Kosmos in kube"
+)
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// KubeNestConfiguration defines the configuration for KubeNest
+type KubeNestConfiguration struct {
+	// TypeMeta contains the API version and kind.
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	KubeNestType KubeNestType `yaml:"kubeNestType" json:"kubeNestType,omitempty"`
+
+	KosmosKubeConfig KosmosKubeConfig `yaml:"kosmosKubeConfig" json:"kosmosKubeConfig,omitempty"`
+
+	KubeInKubeConfig KubeInKubeConfig `yaml:"kubeInKubeConfig" json:"kubeInKubeConfig,omitempty"`
+}
+
+type EtcdCluster struct {
+}
+
+type KosmosKubeConfig struct {
+	// AllowNodeOwnbyMulticluster indicates whether to allow nodes to be owned by multiple clusters.
+	AllowNodeOwnbyMulticluster bool `yaml:"allowNodeOwnbyMulticluster" json:"allowNodeOwnbyMulticluster,omitempty"`
+}
+
+type KubeInKubeConfig struct {
+	// todo Group according to the parameters of apiserver, etcd, coredns, etc.
+
+	ForceDestroy      bool   `yaml:"forceDestroy" json:"forceDestroy,omitempty"`
+	AnpMode           string `yaml:"anpMode" json:"anpMode,omitempty"`
+	AdmissionPlugins  bool   `yaml:"admissionPlugins" json:"admissionPlugins,omitempty"`
+	ApiServerReplicas int    `yaml:"apiServerReplicas" json:"apiServerReplicas,omitempty"`
+	ClusterCIDR       string `yaml:"clusterCIDR" json:"clusterCIDR,omitempty"`
+	ETCDStorageClass  string `yaml:"etcdStorageClass" json:"etcdStorageClass,omitempty"`
+	ETCDUnitSize      string `yaml:"etcdUnitSize" json:"etcdUnitSize,omitempty"`
+
+	//// Etcd contains the configuration for the etcd statefulset.
+	//Etcd EtcdCluster `yaml:"etcd" json:"etcd,omitempty"`
+
+	//// DNS  contains the configuration for the dns server in kubernetes.
+	//DNS DNS `yaml:"dns" json:"dns,omitempty"`
+	//
+	//// Kubernetes contains the configuration for the kubernetes.
+	//Kubernetes Kubernetes `yaml:"kubernetes" json:"kubernetes,omitempty"`
+	//
+	//// Network  contains the configuration for the network in kubernetes cluster.
+	//Network NetworkConfig `yaml:"network" json:"network,omitempty"`
+	//
+	//// Storage contains the configuration for the storage in kubernetes cluster.
+	//Storage StorageConfig `yaml:"storage" json:"storage,omitempty"`
+	//
+	//// Registry contains the configuration for the registry in kubernetes cluster.
+	//Registry RegistryConfig `yaml:"registry" json:"registry,omitempty"`
+}