diff --git a/cmd/kubenest/operator/app/config/config.go b/cmd/kubenest/operator/app/config/config.go new file mode 100644 index 000000000..0d06e06f6 --- /dev/null +++ b/cmd/kubenest/operator/app/config/config.go @@ -0,0 +1,19 @@ +package config + +import ( + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + componentbaseconfig "k8s.io/component-base/config" + + "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" +) + +// Config has all the configurations for kubenest. +type Config struct { + KubeNestOptions v1alpha1.KubeNestConfiguration + Client clientset.Interface + RestConfig *restclient.Config + KubeconfigStream []byte + // LeaderElection is optional. + LeaderElection componentbaseconfig.LeaderElectionConfiguration +} diff --git a/cmd/kubenest/operator/app/operator.go b/cmd/kubenest/operator/app/operator.go index fb67581da..00b0e3b40 100644 --- a/cmd/kubenest/operator/app/operator.go +++ b/cmd/kubenest/operator/app/operator.go @@ -3,17 +3,24 @@ package app import ( "context" "fmt" + "os" "github.com/spf13/cobra" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/client-go/kubernetes" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" cliflag "k8s.io/component-base/cli/flag" "k8s.io/klog/v2" + kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/manager" + "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/config" "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" + "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" "github.com/kosmos.io/kosmos/pkg/kubenest/constants" "github.com/kosmos.io/kosmos/pkg/kubenest/controller" @@ -32,10 +39,7 @@ func NewVirtualClusterOperatorCommand(ctx context.Context) *cobra.Command { Use: "virtual-cluster-operator", Long: `create virtual kubernetes control plane with VirtualCluster`, RunE: func(cmd *cobra.Command, args []string) error { - if errs := opts.Validate(); len(errs) != 0 { - return errs.ToAggregate() - } - if err := run(ctx, opts); err != nil { + if err := runCommand(ctx, opts); err != nil { return err } return nil @@ -56,6 +60,129 @@ func NewVirtualClusterOperatorCommand(ctx context.Context) *cobra.Command { return cmd } +func runCommand(ctx context.Context, opts *options.Options) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + kc, err := SetupConfig(opts) + if err != nil { + return err + } + return run(ctx, kc) +} + +func SetupConfig(opts *options.Options) (*config.Config, error) { + c := &config.Config{} + + var koc v1alpha1.KubeNestConfiguration + if len(opts.ConfigFile) != 0 { + ko, err := loadConfig(opts.ConfigFile) + if err != nil { + return nil, err + } + koc = *ko + } else { + ko := &v1alpha1.KubeNestConfiguration{} + ko.KubeNestType = v1alpha1.KubeInKube + ko.KosmosKubeConfig.AllowNodeOwnbyMulticluster = false + ko.KubeInKubeConfig.ForceDestroy = opts.DeprecatedOptions.KubeInKubeConfig.ForceDestroy + ko.KubeInKubeConfig.ETCDUnitSize = opts.DeprecatedOptions.KubeInKubeConfig.ETCDUnitSize + ko.KubeInKubeConfig.ETCDStorageClass = opts.DeprecatedOptions.KubeInKubeConfig.ETCDStorageClass + ko.KubeInKubeConfig.AdmissionPlugins = opts.DeprecatedOptions.KubeInKubeConfig.AdmissionPlugins + ko.KubeInKubeConfig.AnpMode = opts.DeprecatedOptions.KubeInKubeConfig.AnpMode + ko.KubeInKubeConfig.ApiServerReplicas = opts.DeprecatedOptions.KubeInKubeConfig.ApiServerReplicas + ko.KubeInKubeConfig.ClusterCIDR = opts.DeprecatedOptions.KubeInKubeConfig.ClusterCIDR + + koc = *ko + } + + fillInForDefault(c, koc) + printKubeNestConfiguration(koc) + + kubeconfigStream, err := os.ReadFile(opts.KubernetesOptions.KubeConfig) + if err != nil { + return nil, fmt.Errorf("read kubeconfig file failed: %v", err) + } + + // Prepare kube config. + kubeConfig, err := createKubeConfig(opts) + if err != nil { + return nil, err + } + + // Prepare kube clients. + client, err := createClients(kubeConfig) + if err != nil { + return nil, err + } + + c.KubeconfigStream = kubeconfigStream + c.RestConfig = kubeConfig + c.Client = client + c.LeaderElection = opts.LeaderElection + c.KubeNestOptions = koc + + return c, nil +} + +// TODO +func printKubeNestConfiguration(koc v1alpha1.KubeNestConfiguration) { + +} + +// TODO +func fillInForDefault(c *config.Config, koc v1alpha1.KubeNestConfiguration) { + +} + +func loadConfig(file string) (*v1alpha1.KubeNestConfiguration, error) { + data, err := os.ReadFile(file) + if err != nil { + return nil, err + } + // The UniversalDecoder runs defaulting and returns the internal type by default. + obj, gvk, err := kubeschedulerscheme.Codecs.UniversalDecoder().Decode(data, nil, nil) + if err != nil { + return nil, err + } + if cfgObj, ok := obj.(*v1alpha1.KubeNestConfiguration); ok { + return cfgObj, nil + } + return nil, fmt.Errorf("couldn't decode as KubeNestConfiguration, got %s: ", gvk) +} + +// createClients creates a kube client and an event client from the given kubeConfig +func createClients(kubeConfig *restclient.Config) (clientset.Interface, error) { + client, err := clientset.NewForConfig(kubeConfig) + if err != nil { + return nil, err + } + + return client, nil +} + +// createKubeConfig creates a kubeConfig from the given config and masterOverride. +func createKubeConfig(opts *options.Options) (*restclient.Config, error) { + if len(opts.KubernetesOptions.KubeConfig) == 0 && len(opts.KubernetesOptions.Master) == 0 { + klog.Warning("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.") + } + + // This creates a client, first loading any specified kubeconfig + // file, and then overriding the Master flag, if non-empty. + kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: opts.KubernetesOptions.KubeConfig}, + &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: opts.KubernetesOptions.Master}}).ClientConfig() + if err != nil { + return nil, err + } + + kubeConfig.DisableCompression = true + kubeConfig.QPS = opts.KubernetesOptions.QPS + kubeConfig.Burst = opts.KubernetesOptions.Burst + + return kubeConfig, nil +} + func startEndPointsControllers(mgr manager.Manager) error { coreEndPointsController := endpointscontroller.CoreDNSController{ Client: mgr.GetClient(), @@ -87,36 +214,30 @@ func startEndPointsControllers(mgr manager.Manager) error { return nil } -func run(ctx context.Context, opts *options.Options) error { - config, err := clientcmd.BuildConfigFromFlags(opts.KubernetesOptions.Master, opts.KubernetesOptions.KubeConfig) - if err != nil { - panic(err) - } - config.QPS, config.Burst = opts.KubernetesOptions.QPS, opts.KubernetesOptions.Burst - +func run(ctx context.Context, config *config.Config) error { newscheme := scheme.NewSchema() - err = apiextensionsv1.AddToScheme(newscheme) + err := apiextensionsv1.AddToScheme(newscheme) if err != nil { panic(err) } - mgr, err := controllerruntime.NewManager(config, controllerruntime.Options{ + mgr, err := controllerruntime.NewManager(config.RestConfig, controllerruntime.Options{ Logger: klog.Background(), Scheme: newscheme, - LeaderElection: opts.LeaderElection.LeaderElect, - LeaderElectionID: opts.LeaderElection.ResourceName, - LeaderElectionNamespace: opts.LeaderElection.ResourceNamespace, + LeaderElection: config.LeaderElection.LeaderElect, + LeaderElectionID: config.LeaderElection.ResourceName, + LeaderElectionNamespace: config.LeaderElection.ResourceNamespace, }) if err != nil { return fmt.Errorf("failed to build controller manager: %v", err) } - hostKubeClient, err := kubernetes.NewForConfig(config) + hostKubeClient, err := kubernetes.NewForConfig(config.RestConfig) if err != nil { return fmt.Errorf("could not create clientset: %v", err) } - kosmosClient, err := versioned.NewForConfig(config) + kosmosClient, err := versioned.NewForConfig(config.RestConfig) if err != nil { return fmt.Errorf("could not create clientset: %v", err) } @@ -127,7 +248,7 @@ func run(ctx context.Context, opts *options.Options) error { EventRecorder: mgr.GetEventRecorderFor(constants.InitControllerName), RootClientSet: hostKubeClient, KosmosClient: kosmosClient, - KubeNestOptions: &opts.KubeNestOptions, + KubeNestOptions: &config.KubeNestOptions, } if err = VirtualClusterInitController.SetupWithManager(mgr); err != nil { return fmt.Errorf("error starting %s: %v", constants.InitControllerName, err) @@ -153,19 +274,20 @@ func run(ctx context.Context, opts *options.Options) error { hostKubeClient, mgr.GetEventRecorderFor(constants.NodeControllerName), kosmosClient, - &opts.KubeNestOptions, + &config.KubeNestOptions, ) if err = VirtualClusterNodeController.SetupWithManager(mgr); err != nil { return fmt.Errorf("error starting %s: %v", constants.NodeControllerName, err) } - if opts.KosmosJoinController { + if config.KubeNestOptions.KubeNestType == v1alpha1.KosmosKube { KosmosJoinController := kosmos.KosmosJoinController{ Client: mgr.GetClient(), EventRecorder: mgr.GetEventRecorderFor(constants.KosmosJoinControllerName), - KubeconfigPath: opts.KubernetesOptions.KubeConfig, - AllowNodeOwnbyMulticluster: opts.AllowNodeOwnbyMulticluster, + KubeConfig: config.RestConfig, + KubeconfigStream: config.KubeconfigStream, + AllowNodeOwnbyMulticluster: config.KubeNestOptions.KosmosKubeConfig.AllowNodeOwnbyMulticluster, } if err = KosmosJoinController.SetupWithManager(mgr); err != nil { return fmt.Errorf("error starting %s: %v", constants.KosmosJoinControllerName, err) diff --git a/cmd/kubenest/operator/app/options/options.go b/cmd/kubenest/operator/app/options/options.go index f29396f91..89603c161 100644 --- a/cmd/kubenest/operator/app/options/options.go +++ b/cmd/kubenest/operator/app/options/options.go @@ -5,15 +5,19 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" componentbaseconfig "k8s.io/component-base/config" + "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/utils" ) type Options struct { LeaderElection componentbaseconfig.LeaderElectionConfiguration KubernetesOptions KubernetesOptions - KubeNestOptions KubeNestOptions + DeprecatedOptions v1alpha1.KubeNestConfiguration AllowNodeOwnbyMulticluster bool KosmosJoinController bool + + // ConfigFile is the location of the kubenest's configuration file. + ConfigFile string } type KubernetesOptions struct { @@ -58,11 +62,12 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) { flags.StringVar(&o.KubernetesOptions.Master, "master", "", "Used to generate kubeconfig for downloading, if not specified, will use host in kubeconfig.") flags.BoolVar(&o.AllowNodeOwnbyMulticluster, "multiowner", false, "Allow node own by multicluster or not.") flags.BoolVar(&o.KosmosJoinController, "kosmos-join-controller", false, "Turn on or off kosmos-join-controller.") - flags.BoolVar(&o.KubeNestOptions.ForceDestroy, "kube-nest-force-destroy", false, "Force destroy the node.If it set true.If set to true, Kubernetes will not evict the existing nodes on the node when joining nodes to the tenant's control plane, but will instead force destroy.") - flags.StringVar(&o.KubeNestOptions.AnpMode, "kube-nest-anp-mode", "tcp", "kube-apiserver network proxy mode, must be set to tcp or uds. uds mode the replicas for apiserver should be one, and tcp for multi apiserver replicas.") - flags.BoolVar(&o.KubeNestOptions.AdmissionPlugins, "kube-nest-admission-plugins", false, "kube-apiserver network disable-admission-plugins, false for - --disable-admission-plugins=License, true for remove the --disable-admission-plugins=License flag .") - flags.IntVar(&o.KubeNestOptions.ApiServerReplicas, "kube-nest-apiserver-replicas", 1, "virtual-cluster kube-apiserver replicas. default is 2.") - flags.StringVar(&o.KubeNestOptions.ClusterCIDR, "cluster-cidr", "10.244.0.0/16", "Used to set the cluster-cidr of kube-controller-manager and kube-proxy (configmap)") - flags.StringVar(&o.KubeNestOptions.ETCDStorageClass, "etcd-storage-class", "openebs-hostpath", "Used to set the etcd storage class.") - flags.StringVar(&o.KubeNestOptions.ETCDUnitSize, "etcd-unit-size", "1Gi", "Used to set the etcd unit size, each node is allocated storage of etcd-unit-size.") + flags.BoolVar(&o.DeprecatedOptions.KubeInKubeConfig.ForceDestroy, "kube-nest-force-destroy", false, "Force destroy the node.If it set true.If set to true, Kubernetes will not evict the existing nodes on the node when joining nodes to the tenant's control plane, but will instead force destroy.") + flags.StringVar(&o.DeprecatedOptions.KubeInKubeConfig.AnpMode, "kube-nest-anp-mode", "tcp", "kube-apiserver network proxy mode, must be set to tcp or uds. uds mode the replicas for apiserver should be one, and tcp for multi apiserver replicas.") + flags.BoolVar(&o.DeprecatedOptions.KubeInKubeConfig.AdmissionPlugins, "kube-nest-admission-plugins", false, "kube-apiserver network disable-admission-plugins, false for - --disable-admission-plugins=License, true for remove the --disable-admission-plugins=License flag .") + flags.IntVar(&o.DeprecatedOptions.KubeInKubeConfig.ApiServerReplicas, "kube-nest-apiserver-replicas", 1, "virtual-cluster kube-apiserver replicas. default is 2.") + flags.StringVar(&o.DeprecatedOptions.KubeInKubeConfig.ClusterCIDR, "cluster-cidr", "10.244.0.0/16", "Used to set the cluster-cidr of kube-controller-manager and kube-proxy (configmap)") + flags.StringVar(&o.DeprecatedOptions.KubeInKubeConfig.ETCDStorageClass, "etcd-storage-class", "openebs-hostpath", "Used to set the etcd storage class.") + flags.StringVar(&o.DeprecatedOptions.KubeInKubeConfig.ETCDUnitSize, "etcd-unit-size", "1Gi", "Used to set the etcd unit size, each node is allocated storage of etcd-unit-size.") + flags.StringVar(&o.ConfigFile, "config", "", "The path to the configuration file.") } diff --git a/cmd/kubenest/operator/app/options/validation.go b/cmd/kubenest/operator/app/options/validation.go deleted file mode 100644 index 6d18c85a2..000000000 --- a/cmd/kubenest/operator/app/options/validation.go +++ /dev/null @@ -1,10 +0,0 @@ -package options - -import "k8s.io/apimachinery/pkg/util/validation/field" - -// Validate checks Options and return a slice of found errs. -func (o *Options) Validate() field.ErrorList { - errs := field.ErrorList{} - - return errs -} diff --git a/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go b/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go new file mode 100644 index 000000000..08d69093f --- /dev/null +++ b/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go @@ -0,0 +1,65 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type KubeNestType string + +const ( + KubeInKube KubeNestType = "Kube in kube" + KosmosKube KubeNestType = "Kosmos in kube" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// KubeNestConfiguration defines the configuration for KubeNest +type KubeNestConfiguration struct { + // TypeMeta contains the API version and kind. + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + KubeNestType KubeNestType `yaml:"kubeNestType" json:"kubeNestType,omitempty"` + + KosmosKubeConfig KosmosKubeConfig `yaml:"kosmosKubeConfig" json:"kosmosKubeConfig,omitempty"` + + KubeInKubeConfig KubeInKubeConfig `yaml:"kubeInKubeConfig" json:"kubeInKubeConfig,omitempty"` +} + +type EtcdCluster struct { +} + +type KosmosKubeConfig struct { + // AllowNodeOwnbyMulticluster indicates whether to allow nodes to be owned by multiple clusters. + AllowNodeOwnbyMulticluster bool `yaml:"allowNodeOwnbyMulticluster" json:"allowNodeOwnbyMulticluster,omitempty"` +} + +type KubeInKubeConfig struct { + // todo Group according to the parameters of apiserver, etcd, coredns, etc. + + ForceDestroy bool `yaml:"forceDestroy" json:"forceDestroy,omitempty"` + AnpMode string `yaml:"anpMode" json:"anpMode,omitempty"` + AdmissionPlugins bool `yaml:"admissionPlugins" json:"admissionPlugins,omitempty"` + ApiServerReplicas int `yaml:"apiServerReplicas" json:"apiServerReplicas,omitempty"` + ClusterCIDR string `yaml:"clusterCIDR" json:"clusterCIDR,omitempty"` + ETCDStorageClass string `yaml:"etcdStorageClass" json:"etcdStorageClass,omitempty"` + ETCDUnitSize string `yaml:"etcdUnitSize" json:"etcdUnitSize,omitempty"` + + //// Etcd contains the configuration for the etcd statefulset. + //Etcd EtcdCluster `yaml:"etcd" json:"etcd,omitempty"` + + //// DNS contains the configuration for the dns server in kubernetes. + //DNS DNS `yaml:"dns" json:"dns,omitempty"` + // + //// Kubernetes contains the configuration for the kubernetes. + //Kubernetes Kubernetes `yaml:"kubernetes" json:"kubernetes,omitempty"` + // + //// Network contains the configuration for the network in kubernetes cluster. + //Network NetworkConfig `yaml:"network" json:"network,omitempty"` + // + //// Storage contains the configuration for the storage in kubernetes cluster. + //Storage StorageConfig `yaml:"storage" json:"storage,omitempty"` + // + //// Registry contains the configuration for the registry in kubernetes cluster. + //Registry RegistryConfig `yaml:"registry" json:"registry,omitempty"` +} diff --git a/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go index 8e4c323f3..f65fafbf1 100644 --- a/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go @@ -843,6 +843,22 @@ func (in *DistributionSpec) DeepCopy() *DistributionSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdCluster) DeepCopyInto(out *EtcdCluster) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdCluster. +func (in *EtcdCluster) DeepCopy() *EtcdCluster { + if in == nil { + return nil + } + out := new(EtcdCluster) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Fdb) DeepCopyInto(out *Fdb) { *out = *in @@ -1021,6 +1037,66 @@ func (in *Iptables) DeepCopy() *Iptables { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KosmosKubeConfig) DeepCopyInto(out *KosmosKubeConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KosmosKubeConfig. +func (in *KosmosKubeConfig) DeepCopy() *KosmosKubeConfig { + if in == nil { + return nil + } + out := new(KosmosKubeConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeInKubeConfig) DeepCopyInto(out *KubeInKubeConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeInKubeConfig. +func (in *KubeInKubeConfig) DeepCopy() *KubeInKubeConfig { + if in == nil { + return nil + } + out := new(KubeInKubeConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeNestConfiguration) DeepCopyInto(out *KubeNestConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.KosmosKubeConfig = in.KosmosKubeConfig + out.KubeInKubeConfig = in.KubeInKubeConfig + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeNestConfiguration. +func (in *KubeNestConfiguration) DeepCopy() *KubeNestConfiguration { + if in == nil { + return nil + } + out := new(KubeNestConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KubeNestConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LeafModel) DeepCopyInto(out *LeafModel) { *out = *in diff --git a/pkg/apis/kosmos/v1alpha1/zz_generated.register.go b/pkg/apis/kosmos/v1alpha1/zz_generated.register.go index 657efe95b..49a6d7652 100644 --- a/pkg/apis/kosmos/v1alpha1/zz_generated.register.go +++ b/pkg/apis/kosmos/v1alpha1/zz_generated.register.go @@ -56,6 +56,7 @@ func addKnownTypes(scheme *runtime.Scheme) error { &DistributionPolicyList{}, &GlobalNode{}, &GlobalNodeList{}, + &KubeNestConfiguration{}, &NodeConfig{}, &NodeConfigList{}, &PodConvertPolicy{}, diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 895b4202a..9eab67a6e 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -45,6 +45,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.DistributionPolicy": schema_pkg_apis_kosmos_v1alpha1_DistributionPolicy(ref), "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.DistributionPolicyList": schema_pkg_apis_kosmos_v1alpha1_DistributionPolicyList(ref), "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.DistributionSpec": schema_pkg_apis_kosmos_v1alpha1_DistributionSpec(ref), + "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.EtcdCluster": schema_pkg_apis_kosmos_v1alpha1_EtcdCluster(ref), "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.Fdb": schema_pkg_apis_kosmos_v1alpha1_Fdb(ref), "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.GlobalNode": schema_pkg_apis_kosmos_v1alpha1_GlobalNode(ref), "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.GlobalNodeList": schema_pkg_apis_kosmos_v1alpha1_GlobalNodeList(ref), @@ -53,6 +54,9 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.HostAliasesConverter": schema_pkg_apis_kosmos_v1alpha1_HostAliasesConverter(ref), "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.HostPath": schema_pkg_apis_kosmos_v1alpha1_HostPath(ref), "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.Iptables": schema_pkg_apis_kosmos_v1alpha1_Iptables(ref), + "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.KosmosKubeConfig": schema_pkg_apis_kosmos_v1alpha1_KosmosKubeConfig(ref), + "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.KubeInKubeConfig": schema_pkg_apis_kosmos_v1alpha1_KubeInKubeConfig(ref), + "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.KubeNestConfiguration": schema_pkg_apis_kosmos_v1alpha1_KubeNestConfiguration(ref), "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.LeafModel": schema_pkg_apis_kosmos_v1alpha1_LeafModel(ref), "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.LeafNodeItem": schema_pkg_apis_kosmos_v1alpha1_LeafNodeItem(ref), "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.NICNodeNames": schema_pkg_apis_kosmos_v1alpha1_NICNodeNames(ref), @@ -1590,6 +1594,16 @@ func schema_pkg_apis_kosmos_v1alpha1_DistributionSpec(ref common.ReferenceCallba } } +func schema_pkg_apis_kosmos_v1alpha1_EtcdCluster(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + }, + }, + } +} + func schema_pkg_apis_kosmos_v1alpha1_Fdb(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -1887,6 +1901,132 @@ func schema_pkg_apis_kosmos_v1alpha1_Iptables(ref common.ReferenceCallback) comm } } +func schema_pkg_apis_kosmos_v1alpha1_KosmosKubeConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "allowNodeOwnbyMulticluster": { + SchemaProps: spec.SchemaProps{ + Description: "AllowNodeOwnbyMulticluster indicates whether to allow nodes to be owned by multiple clusters.", + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_kosmos_v1alpha1_KubeInKubeConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "forceDestroy": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, + "anpMode": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "admissionPlugins": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, + "apiServerReplicas": { + SchemaProps: spec.SchemaProps{ + Type: []string{"integer"}, + Format: "int32", + }, + }, + "clusterCIDR": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "etcdStorageClass": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "etcdUnitSize": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_kosmos_v1alpha1_KubeNestConfiguration(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "KubeNestConfiguration defines the configuration for KubeNest", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "kubeNestType": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "kosmosKubeConfig": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.KosmosKubeConfig"), + }, + }, + "kubeInKubeConfig": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.KubeInKubeConfig"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.KosmosKubeConfig", "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1.KubeInKubeConfig", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + } +} + func schema_pkg_apis_kosmos_v1alpha1_LeafModel(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/pkg/kubenest/controller/kosmos/kosmos_join_controller.go b/pkg/kubenest/controller/kosmos/kosmos_join_controller.go index d30eac7b1..becfca6c3 100644 --- a/pkg/kubenest/controller/kosmos/kosmos_join_controller.go +++ b/pkg/kubenest/controller/kosmos/kosmos_join_controller.go @@ -14,6 +14,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" @@ -41,12 +42,12 @@ import ( type KosmosJoinController struct { client.Client EventRecorder record.EventRecorder - KubeconfigPath string + KubeConfig *restclient.Config KubeconfigStream []byte AllowNodeOwnbyMulticluster bool } -var nodeOwnerMap map[string]string = make(map[string]string) +var nodeOwnerMap = make(map[string]string) var mu sync.Mutex var once sync.Once @@ -571,11 +572,6 @@ func (c *KosmosJoinController) SetupWithManager(mgr manager.Manager) error { // skip reservedNS return obj.GetNamespace() != utils.ReservedNS } - kubeconfigStream, err := os.ReadFile(c.KubeconfigPath) - if err != nil { - return fmt.Errorf("read kubeconfig file failed: %v", err) - } - c.KubeconfigStream = kubeconfigStream return ctrl.NewControllerManagedBy(mgr). Named(constants.KosmosJoinControllerName). diff --git a/pkg/kubenest/controller/virtualcluster.node.controller/node_controller.go b/pkg/kubenest/controller/virtualcluster.node.controller/node_controller.go index 964dca046..68ab0b65e 100644 --- a/pkg/kubenest/controller/virtualcluster.node.controller/node_controller.go +++ b/pkg/kubenest/controller/virtualcluster.node.controller/node_controller.go @@ -23,7 +23,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" "github.com/kosmos.io/kosmos/pkg/kubenest/constants" @@ -39,17 +38,17 @@ type NodeController struct { RootClientSet kubernetes.Interface EventRecorder record.EventRecorder KosmosClient versioned.Interface - Options *options.KubeNestOptions + Options *v1alpha1.KubeNestConfiguration sem chan struct{} } -func NewNodeController(client client.Client, RootClientSet kubernetes.Interface, EventRecorder record.EventRecorder, KosmosClient versioned.Interface, Options *options.KubeNestOptions) *NodeController { +func NewNodeController(client client.Client, RootClientSet kubernetes.Interface, EventRecorder record.EventRecorder, KosmosClient versioned.Interface, options *v1alpha1.KubeNestConfiguration) *NodeController { r := NodeController{ Client: client, RootClientSet: RootClientSet, EventRecorder: EventRecorder, KosmosClient: KosmosClient, - Options: Options, + Options: options, sem: make(chan struct{}, env.GetNodeTaskMaxGoroutines()), } return &r @@ -308,7 +307,6 @@ func (r *NodeController) cleanGlobalNode(ctx context.Context, nodeInfos []v1alph HostClient: r.Client, HostK8sClient: r.RootClientSet, Opt: r.Options, - // VirtualK8sClient: _, }) }) } diff --git a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go index 8bd159145..8269fd821 100644 --- a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go +++ b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go @@ -17,7 +17,6 @@ import ( "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/kubenest/constants" env "github.com/kosmos.io/kosmos/pkg/kubenest/controller/virtualcluster.node.controller/env" @@ -34,7 +33,7 @@ type TaskOpt struct { HostK8sClient kubernetes.Interface VirtualK8sClient kubernetes.Interface - Opt *options.KubeNestOptions + Opt *v1alpha1.KubeNestConfiguration logger *PrefixedLogger } @@ -100,7 +99,7 @@ func NewDrainHostNodeTask() Task { Retry: true, Skip: func(ctx context.Context, opt TaskOpt) bool { if opt.Opt != nil { - return opt.Opt.ForceDestroy + return opt.Opt.KubeInKubeConfig.ForceDestroy } return false }, @@ -129,7 +128,7 @@ func NewDrainVirtualNodeTask() Task { // ErrorIgnore: true, Skip: func(ctx context.Context, opt TaskOpt) bool { if opt.Opt != nil { - return opt.Opt.ForceDestroy + return opt.Opt.KubeInKubeConfig.ForceDestroy } return false }, diff --git a/pkg/kubenest/controller/virtualcluster_execute_controller.go b/pkg/kubenest/controller/virtualcluster_execute_controller.go index 085eadd12..11c7478ac 100644 --- a/pkg/kubenest/controller/virtualcluster_execute_controller.go +++ b/pkg/kubenest/controller/virtualcluster_execute_controller.go @@ -8,7 +8,6 @@ import ( "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" - ko "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/kubenest" "github.com/kosmos.io/kosmos/pkg/kubenest/constants" @@ -22,7 +21,7 @@ type Executor struct { config *rest.Config } -func NewExecutor(virtualCluster *v1alpha1.VirtualCluster, c client.Client, config *rest.Config, kubeNestOptions *ko.KubeNestOptions) (*Executor, error) { +func NewExecutor(virtualCluster *v1alpha1.VirtualCluster, c client.Client, config *rest.Config, kubeNestOptions *v1alpha1.KubeNestConfiguration) (*Executor, error) { var phase *workflow.Phase opts := []kubenest.InitOpt{ @@ -71,7 +70,7 @@ func (e *Executor) Execute() error { // return err // } // -// kubeconfigBytes := secret.Data[constants.KubeConfig] +// kubeconfigBytes := secret.Data[constants.restConfig] // configString := base64.StdEncoding.EncodeToString(kubeconfigBytes) // e.virtualCluster.Spec.Kubeconfig = configString // e.virtualCluster.Status.Phase = v1alpha1.Completed diff --git a/pkg/kubenest/controller/virtualcluster_init_controller.go b/pkg/kubenest/controller/virtualcluster_init_controller.go index ff0132777..88ef52e77 100644 --- a/pkg/kubenest/controller/virtualcluster_init_controller.go +++ b/pkg/kubenest/controller/virtualcluster_init_controller.go @@ -31,7 +31,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" "github.com/kosmos.io/kosmos/pkg/kubenest/constants" @@ -45,7 +44,7 @@ type VirtualClusterInitController struct { RootClientSet kubernetes.Interface KosmosClient versioned.Interface lock sync.Mutex - KubeNestOptions *options.KubeNestOptions + KubeNestOptions *v1alpha1.KubeNestConfiguration } type NodePool struct { @@ -267,7 +266,7 @@ func (c *VirtualClusterInitController) removeFinalizer(virtualCluster *v1alpha1. } // createVirtualCluster assign work nodes, create control plane and create compoennts from manifests -func (c *VirtualClusterInitController) createVirtualCluster(virtualCluster *v1alpha1.VirtualCluster, kubeNestOptions *options.KubeNestOptions) error { +func (c *VirtualClusterInitController) createVirtualCluster(virtualCluster *v1alpha1.VirtualCluster, kubeNestOptions *v1alpha1.KubeNestConfiguration) error { klog.V(2).Infof("Reconciling virtual cluster", "name", virtualCluster.Name) //Assign host port diff --git a/pkg/kubenest/controlplane/apiserver.go b/pkg/kubenest/controlplane/apiserver.go index c6c6f2074..c9f4e48b5 100644 --- a/pkg/kubenest/controlplane/apiserver.go +++ b/pkg/kubenest/controlplane/apiserver.go @@ -8,14 +8,14 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" clientset "k8s.io/client-go/kubernetes" - "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" + "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/kubenest/constants" "github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/apiserver" "github.com/kosmos.io/kosmos/pkg/kubenest/util" ) -func EnsureVirtualClusterAPIServer(client clientset.Interface, name, namespace string, portMap map[string]int32, opt *options.KubeNestOptions) error { - if err := installAPIServer(client, name, namespace, portMap, opt); err != nil { +func EnsureVirtualClusterAPIServer(client clientset.Interface, name, namespace string, portMap map[string]int32, kubeNestConfiguration *v1alpha1.KubeNestConfiguration) error { + if err := installAPIServer(client, name, namespace, portMap, kubeNestConfiguration); err != nil { return fmt.Errorf("failed to install virtual cluster apiserver, err: %w", err) } return nil @@ -29,7 +29,7 @@ func DeleteVirtualClusterAPIServer(client clientset.Interface, name, namespace s return nil } -func installAPIServer(client clientset.Interface, name, namespace string, portMap map[string]int32, opt *options.KubeNestOptions) error { +func installAPIServer(client clientset.Interface, name, namespace string, portMap map[string]int32, kubeNestConfiguration *v1alpha1.KubeNestConfiguration) error { imageRepository, imageVersion := util.GetImageMessage() clusterIp, err := util.GetEtcdServiceClusterIp(namespace, name+constants.EtcdSuffix, client) if err != nil { @@ -61,11 +61,11 @@ func installAPIServer(client clientset.Interface, name, namespace string, portMa ServiceSubnet: constants.ApiServerServiceSubnet, VirtualClusterCertsSecret: fmt.Sprintf("%s-%s", name, "cert"), EtcdCertsSecret: fmt.Sprintf("%s-%s", name, "etcd-cert"), - Replicas: opt.ApiServerReplicas, + Replicas: kubeNestConfiguration.KubeInKubeConfig.ApiServerReplicas, EtcdListenClientPort: constants.ApiServerEtcdListenClientPort, ClusterPort: portMap[constants.ApiServerPortKey], - AdmissionPlugins: opt.AdmissionPlugins, IPV6First: IPV6FirstFlag, + AdmissionPlugins: kubeNestConfiguration.KubeInKubeConfig.AdmissionPlugins, }) if err != nil { return fmt.Errorf("error when parsing virtual cluster apiserver deployment template: %w", err) diff --git a/pkg/kubenest/controlplane/etcd.go b/pkg/kubenest/controlplane/etcd.go index c3167aa0d..39c9b4438 100644 --- a/pkg/kubenest/controlplane/etcd.go +++ b/pkg/kubenest/controlplane/etcd.go @@ -12,15 +12,14 @@ import ( "k8s.io/component-base/cli/flag" "k8s.io/klog" - ko "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/kubenest/constants" "github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/etcd" "github.com/kosmos.io/kosmos/pkg/kubenest/util" ) -func EnsureVirtualClusterEtcd(client clientset.Interface, name, namespace string, ko *ko.KubeNestOptions, vc *v1alpha1.VirtualCluster) error { - if err := installEtcd(client, name, namespace, ko, vc); err != nil { +func EnsureVirtualClusterEtcd(client clientset.Interface, name, namespace string, kubeNestConfiguration *v1alpha1.KubeNestConfiguration, vc *v1alpha1.VirtualCluster) error { + if err := installEtcd(client, name, namespace, kubeNestConfiguration, vc); err != nil { return err } return nil @@ -34,13 +33,13 @@ func DeleteVirtualClusterEtcd(client clientset.Interface, name, namespace string return nil } -func installEtcd(client clientset.Interface, name, namespace string, ko *ko.KubeNestOptions, vc *v1alpha1.VirtualCluster) error { +func installEtcd(client clientset.Interface, name, namespace string, kubeNestConfiguration *v1alpha1.KubeNestConfiguration, vc *v1alpha1.VirtualCluster) error { imageRepository, imageVersion := util.GetImageMessage() nodeCount := getNodeCountFromPromotePolicy(vc) - resourceQuantity, err := resource.ParseQuantity(ko.ETCDUnitSize) + resourceQuantity, err := resource.ParseQuantity(kubeNestConfiguration.KubeInKubeConfig.ETCDUnitSize) if err != nil { - klog.Errorf("Failed to parse quantity %s: %v", ko.ETCDUnitSize, err) + klog.Errorf("Failed to parse quantity %s: %v", kubeNestConfiguration.KubeInKubeConfig.ETCDUnitSize, err) return err } resourceQuantity.Set(resourceQuantity.Value() * int64(nodeCount)) @@ -78,7 +77,6 @@ func installEtcd(client clientset.Interface, name, namespace string, ko *ko.Kube Namespace: namespace, ImageRepository: imageRepository, Version: imageVersion, - VirtualControllerLabel: vclabel, EtcdClientService: fmt.Sprintf("%s-%s", name, "etcd-client"), CertsSecretName: fmt.Sprintf("%s-%s", name, "etcd-cert"), EtcdPeerServiceName: fmt.Sprintf("%s-%s", name, "etcd"), @@ -88,8 +86,9 @@ func installEtcd(client clientset.Interface, name, namespace string, ko *ko.Kube Replicas: constants.EtcdReplicas, EtcdListenClientPort: constants.EtcdListenClientPort, EtcdListenPeerPort: constants.EtcdListenPeerPort, - ETCDStorageClass: ko.ETCDStorageClass, + ETCDStorageClass: kubeNestConfiguration.KubeInKubeConfig.ETCDStorageClass, ETCDStorageSize: resourceQuantity.String(), + VirtualControllerLabel: vclabel, IPV6First: IPV6FirstFlag, }) if err != nil { diff --git a/pkg/kubenest/init.go b/pkg/kubenest/init.go index 2fd8b886a..aa9585b3f 100644 --- a/pkg/kubenest/init.go +++ b/pkg/kubenest/init.go @@ -10,7 +10,6 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - ko "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" "github.com/kosmos.io/kosmos/pkg/kubenest/tasks" @@ -37,7 +36,7 @@ type initData struct { externalIP string hostPort int32 hostPortMap map[string]int32 - kubeNestOptions *ko.KubeNestOptions + kubeNestOptions *v1alpha1.KubeNestConfiguration virtualCluster *v1alpha1.VirtualCluster ETCDStorageClass string ETCDUnitSize string @@ -50,7 +49,7 @@ type InitOptions struct { virtualClusterVersion string virtualClusterDataDir string virtualCluster *v1alpha1.VirtualCluster - KubeNestOptions *ko.KubeNestOptions + KubeNestOptions *v1alpha1.KubeNestConfiguration } func NewInitPhase(opts *InitOptions) *workflow.Phase { @@ -132,7 +131,7 @@ func NewInitOptWithKubeconfig(config *rest.Config) InitOpt { } } -func NewInitOptWithKubeNestOptions(options *ko.KubeNestOptions) InitOpt { +func NewInitOptWithKubeNestOptions(options *v1alpha1.KubeNestConfiguration) InitOpt { return func(o *InitOptions) { o.KubeNestOptions = options } @@ -192,8 +191,8 @@ func newRunData(opt *InitOptions) (*initData, error) { hostPortMap: opt.virtualCluster.Status.PortMap, kubeNestOptions: opt.KubeNestOptions, virtualCluster: opt.virtualCluster, - ETCDUnitSize: opt.KubeNestOptions.ETCDUnitSize, - ETCDStorageClass: opt.KubeNestOptions.ETCDStorageClass, + ETCDUnitSize: opt.KubeNestOptions.KubeInKubeConfig.ETCDUnitSize, + ETCDStorageClass: opt.KubeNestOptions.KubeInKubeConfig.ETCDStorageClass, }, nil } @@ -265,7 +264,7 @@ func (i initData) DynamicClient() *dynamic.DynamicClient { return i.dynamicClient } -func (i initData) KubeNestOpt() *ko.KubeNestOptions { +func (i initData) KubeNestOpt() *v1alpha1.KubeNestConfiguration { return i.kubeNestOptions } diff --git a/pkg/kubenest/tasks/anp.go b/pkg/kubenest/tasks/anp.go index f943e0ee0..60eee0313 100644 --- a/pkg/kubenest/tasks/anp.go +++ b/pkg/kubenest/tasks/anp.go @@ -16,7 +16,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" - ko "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" + "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/kubenest/constants" "github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/apiserver" "github.com/kosmos.io/kosmos/pkg/kubenest/util" @@ -80,7 +80,7 @@ func runAnpServer(r workflow.RunData) error { Name: name, ProxyServerPort: portMap[constants.ApiServerNetworkProxyServerPortKey], SvcName: fmt.Sprintf("%s-konnectivity-server.%s.svc.cluster.local", name, namespace), - AnpMode: kubeNestOpt.AnpMode, + AnpMode: kubeNestOpt.KubeInKubeConfig.AnpMode, }) if err != nil { return fmt.Errorf("failed to parse egress_selector_configuration config map template, err: %w", err) @@ -147,7 +147,7 @@ func uninstallAnp(r workflow.RunData) error { klog.V(2).InfoS("[VirtualClusterAnp] Successfully uninstalled virtual cluster anp component", "virtual cluster", klog.KObj(data)) return util.ForEachObjectInYAML(context.TODO(), vcClient, []byte(anpManifest), "", actionFunc) } -func installAnpServer(client clientset.Interface, name, namespace string, portMap map[string]int32, kubeNestOpt *ko.KubeNestOptions) error { +func installAnpServer(client clientset.Interface, name, namespace string, portMap map[string]int32, kubeNestConfiguration *v1alpha1.KubeNestConfiguration) error { imageRepository, imageVersion := util.GetImageMessage() clusterIp, err := util.GetEtcdServiceClusterIp(namespace, name+constants.EtcdSuffix, client) if err != nil { @@ -186,7 +186,7 @@ func installAnpServer(client clientset.Interface, name, namespace string, portMa ServiceSubnet: constants.ApiServerServiceSubnet, VirtualClusterCertsSecret: fmt.Sprintf("%s-%s", name, "cert"), EtcdCertsSecret: fmt.Sprintf("%s-%s", name, "etcd-cert"), - Replicas: kubeNestOpt.ApiServerReplicas, + Replicas: kubeNestConfiguration.KubeInKubeConfig.ApiServerReplicas, EtcdListenClientPort: constants.ApiServerEtcdListenClientPort, ClusterPort: portMap[constants.ApiServerPortKey], AgentPort: portMap[constants.ApiServerNetworkProxyAgentPortKey], @@ -195,8 +195,8 @@ func installAnpServer(client clientset.Interface, name, namespace string, portMa AdminPort: portMap[constants.ApiServerNetworkProxyAdminPortKey], KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config-clusterip"), Name: name, - AnpMode: kubeNestOpt.AnpMode, - AdmissionPlugins: kubeNestOpt.AdmissionPlugins, + AnpMode: kubeNestConfiguration.KubeInKubeConfig.AnpMode, + AdmissionPlugins: kubeNestConfiguration.KubeInKubeConfig.AdmissionPlugins, IPV6First: IPV6FirstFlag, }) if err != nil { @@ -234,7 +234,7 @@ func installAnpAgent(data InitData) error { return util.ForEachObjectInYAML(context.TODO(), vcClient, []byte(anpAgentManifestBytes), "", actionFunc) } -func getAnpAgentManifest(client clientset.Interface, name string, namespace string, portMap map[string]int32, kubeNestOpt *ko.KubeNestOptions) (string, dynamic.Interface, error) { +func getAnpAgentManifest(client clientset.Interface, name string, namespace string, portMap map[string]int32, kubeNestConfiguration *v1alpha1.KubeNestConfiguration) (string, dynamic.Interface, error) { imageRepository, imageVersion := util.GetImageMessage() // get apiServer hostIp proxyServerHost, err := getDeploymentPodIPs(client, namespace, fmt.Sprintf("%s-%s", name, "apiserver")) @@ -256,7 +256,7 @@ func getAnpAgentManifest(client clientset.Interface, name string, namespace stri Version: imageVersion, AgentPort: portMap[constants.ApiServerNetworkProxyAgentPortKey], ProxyServerHost: proxyServerHost, - AnpMode: kubeNestOpt.AnpMode, + AnpMode: kubeNestConfiguration.KubeInKubeConfig.AnpMode, AgentCertName: fmt.Sprintf("%s-%s", name, "cert"), }) if err != nil { diff --git a/pkg/kubenest/tasks/component.go b/pkg/kubenest/tasks/component.go index 5c4102db1..9f04a5709 100644 --- a/pkg/kubenest/tasks/component.go +++ b/pkg/kubenest/tasks/component.go @@ -54,7 +54,7 @@ func runComponentSubTask(component string) func(r workflow.RunData) error { data.GetName(), data.GetNamespace(), data.RemoteClient(), - kubeNestOpt.ClusterCIDR, + kubeNestOpt.KubeInKubeConfig.ClusterCIDR, ) if err != nil { return fmt.Errorf("failed to apply component %s, err: %w", component, err) diff --git a/pkg/kubenest/tasks/data.go b/pkg/kubenest/tasks/data.go index edf63616f..0336773c8 100644 --- a/pkg/kubenest/tasks/data.go +++ b/pkg/kubenest/tasks/data.go @@ -4,7 +4,6 @@ import ( "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" - ko "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options" "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" "github.com/kosmos.io/kosmos/pkg/kubenest/util/cert" @@ -24,6 +23,6 @@ type InitData interface { HostPort() int32 HostPortMap() map[string]int32 DynamicClient() *dynamic.DynamicClient - KubeNestOpt() *ko.KubeNestOptions + KubeNestOpt() *v1alpha1.KubeNestConfiguration PluginOptions() map[string]string } diff --git a/pkg/kubenest/tasks/proxy.go b/pkg/kubenest/tasks/proxy.go index 3f2043f37..50dd9e1a4 100644 --- a/pkg/kubenest/tasks/proxy.go +++ b/pkg/kubenest/tasks/proxy.go @@ -78,7 +78,7 @@ func runVirtualClusterProxy(r workflow.RunData) error { err = controlplane.EnsureVirtualClusterProxy( virtualClient, kubeconfigString, - kubeNestOpt.ClusterCIDR, + kubeNestOpt.KubeInKubeConfig.ClusterCIDR, ) if err != nil { return fmt.Errorf("failed to install virtual cluster proxy component, err: %w", err) diff --git a/pkg/scheme/scheme.go b/pkg/scheme/scheme.go index ee7157e3c..44b63c09b 100644 --- a/pkg/scheme/scheme.go +++ b/pkg/scheme/scheme.go @@ -3,11 +3,9 @@ package scheme import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" mcsv1alpha1 "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" - clusterlinkv1alpha1 "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" + kosmosv1alpha1 "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1" ) // aggregatedScheme aggregates Kubernetes and extended schemes. @@ -18,7 +16,7 @@ func init() { if err != nil { panic(err) } - err = clusterlinkv1alpha1.AddToScheme(aggregatedScheme) // add clusterlink schemes + err = kosmosv1alpha1.AddToScheme(aggregatedScheme) // add clusterlink schemes if err != nil { panic(err) } @@ -32,20 +30,3 @@ func init() { func NewSchema() *runtime.Scheme { return aggregatedScheme } - -// NewForConfig creates a new client for the given config. -func NewForConfig(config *rest.Config) (client.Client, error) { - return client.New(config, client.Options{ - Scheme: aggregatedScheme, - }) -} - -// NewForConfigOrDie creates a new client for the given config and -// panics if there is an error in the config. -func NewForConfigOrDie(config *rest.Config) client.Client { - c, err := NewForConfig(config) - if err != nil { - panic(err) - } - return c -}