Skip to content

Commit

Permalink
Merge pull request #642 from duanmengkk/feature_kubenest
Browse files Browse the repository at this point in the history
support --configfile for kubenest
duanmengkk authored Jul 3, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
2 parents 1811f3e + d8e07b9 commit c65778d
Showing 21 changed files with 504 additions and 117 deletions.
19 changes: 19 additions & 0 deletions cmd/kubenest/operator/app/config/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package config

import (
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
componentbaseconfig "k8s.io/component-base/config"

"github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1"
)

// Config has all the configurations for kubenest.
type Config struct {
KubeNestOptions v1alpha1.KubeNestConfiguration
Client clientset.Interface
RestConfig *restclient.Config
KubeconfigStream []byte
// LeaderElection is optional.
LeaderElection componentbaseconfig.LeaderElectionConfiguration
}
168 changes: 145 additions & 23 deletions cmd/kubenest/operator/app/operator.go
Original file line number Diff line number Diff line change
@@ -3,17 +3,24 @@ package app
import (
"context"
"fmt"
"os"

"github.com/spf13/cobra"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/client-go/kubernetes"
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog/v2"
kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme"
controllerruntime "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/manager"

"github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/config"
"github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options"
"github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1"
"github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned"
"github.com/kosmos.io/kosmos/pkg/kubenest/constants"
"github.com/kosmos.io/kosmos/pkg/kubenest/controller"
@@ -32,10 +39,7 @@ func NewVirtualClusterOperatorCommand(ctx context.Context) *cobra.Command {
Use: "virtual-cluster-operator",
Long: `create virtual kubernetes control plane with VirtualCluster`,
RunE: func(cmd *cobra.Command, args []string) error {
if errs := opts.Validate(); len(errs) != 0 {
return errs.ToAggregate()
}
if err := run(ctx, opts); err != nil {
if err := runCommand(ctx, opts); err != nil {
return err
}
return nil
@@ -56,6 +60,129 @@ func NewVirtualClusterOperatorCommand(ctx context.Context) *cobra.Command {
return cmd
}

func runCommand(ctx context.Context, opts *options.Options) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()

kc, err := SetupConfig(opts)
if err != nil {
return err
}
return run(ctx, kc)
}

func SetupConfig(opts *options.Options) (*config.Config, error) {
c := &config.Config{}

var koc v1alpha1.KubeNestConfiguration
if len(opts.ConfigFile) != 0 {
ko, err := loadConfig(opts.ConfigFile)
if err != nil {
return nil, err
}
koc = *ko
} else {
ko := &v1alpha1.KubeNestConfiguration{}
ko.KubeNestType = v1alpha1.KubeInKube
ko.KosmosKubeConfig.AllowNodeOwnbyMulticluster = false
ko.KubeInKubeConfig.ForceDestroy = opts.DeprecatedOptions.KubeInKubeConfig.ForceDestroy
ko.KubeInKubeConfig.ETCDUnitSize = opts.DeprecatedOptions.KubeInKubeConfig.ETCDUnitSize
ko.KubeInKubeConfig.ETCDStorageClass = opts.DeprecatedOptions.KubeInKubeConfig.ETCDStorageClass
ko.KubeInKubeConfig.AdmissionPlugins = opts.DeprecatedOptions.KubeInKubeConfig.AdmissionPlugins
ko.KubeInKubeConfig.AnpMode = opts.DeprecatedOptions.KubeInKubeConfig.AnpMode
ko.KubeInKubeConfig.ApiServerReplicas = opts.DeprecatedOptions.KubeInKubeConfig.ApiServerReplicas
ko.KubeInKubeConfig.ClusterCIDR = opts.DeprecatedOptions.KubeInKubeConfig.ClusterCIDR

koc = *ko
}

fillInForDefault(c, koc)
printKubeNestConfiguration(koc)

kubeconfigStream, err := os.ReadFile(opts.KubernetesOptions.KubeConfig)
if err != nil {
return nil, fmt.Errorf("read kubeconfig file failed: %v", err)
}

// Prepare kube config.
kubeConfig, err := createKubeConfig(opts)
if err != nil {
return nil, err
}

// Prepare kube clients.
client, err := createClients(kubeConfig)
if err != nil {
return nil, err
}

c.KubeconfigStream = kubeconfigStream
c.RestConfig = kubeConfig
c.Client = client
c.LeaderElection = opts.LeaderElection
c.KubeNestOptions = koc

return c, nil
}

// TODO
func printKubeNestConfiguration(koc v1alpha1.KubeNestConfiguration) {

}

// TODO
func fillInForDefault(c *config.Config, koc v1alpha1.KubeNestConfiguration) {

}

func loadConfig(file string) (*v1alpha1.KubeNestConfiguration, error) {
data, err := os.ReadFile(file)
if err != nil {
return nil, err
}
// The UniversalDecoder runs defaulting and returns the internal type by default.
obj, gvk, err := kubeschedulerscheme.Codecs.UniversalDecoder().Decode(data, nil, nil)
if err != nil {
return nil, err
}
if cfgObj, ok := obj.(*v1alpha1.KubeNestConfiguration); ok {
return cfgObj, nil
}
return nil, fmt.Errorf("couldn't decode as KubeNestConfiguration, got %s: ", gvk)
}

// createClients creates a kube client and an event client from the given kubeConfig
func createClients(kubeConfig *restclient.Config) (clientset.Interface, error) {
client, err := clientset.NewForConfig(kubeConfig)
if err != nil {
return nil, err
}

return client, nil
}

// createKubeConfig creates a kubeConfig from the given config and masterOverride.
func createKubeConfig(opts *options.Options) (*restclient.Config, error) {
if len(opts.KubernetesOptions.KubeConfig) == 0 && len(opts.KubernetesOptions.Master) == 0 {
klog.Warning("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.")
}

// This creates a client, first loading any specified kubeconfig
// file, and then overriding the Master flag, if non-empty.
kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
&clientcmd.ClientConfigLoadingRules{ExplicitPath: opts.KubernetesOptions.KubeConfig},
&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: opts.KubernetesOptions.Master}}).ClientConfig()
if err != nil {
return nil, err
}

kubeConfig.DisableCompression = true
kubeConfig.QPS = opts.KubernetesOptions.QPS
kubeConfig.Burst = opts.KubernetesOptions.Burst

return kubeConfig, nil
}

func startEndPointsControllers(mgr manager.Manager) error {
coreEndPointsController := endpointscontroller.CoreDNSController{
Client: mgr.GetClient(),
@@ -87,36 +214,30 @@ func startEndPointsControllers(mgr manager.Manager) error {
return nil
}

func run(ctx context.Context, opts *options.Options) error {
config, err := clientcmd.BuildConfigFromFlags(opts.KubernetesOptions.Master, opts.KubernetesOptions.KubeConfig)
if err != nil {
panic(err)
}
config.QPS, config.Burst = opts.KubernetesOptions.QPS, opts.KubernetesOptions.Burst

func run(ctx context.Context, config *config.Config) error {
newscheme := scheme.NewSchema()
err = apiextensionsv1.AddToScheme(newscheme)
err := apiextensionsv1.AddToScheme(newscheme)
if err != nil {
panic(err)
}

mgr, err := controllerruntime.NewManager(config, controllerruntime.Options{
mgr, err := controllerruntime.NewManager(config.RestConfig, controllerruntime.Options{
Logger: klog.Background(),
Scheme: newscheme,
LeaderElection: opts.LeaderElection.LeaderElect,
LeaderElectionID: opts.LeaderElection.ResourceName,
LeaderElectionNamespace: opts.LeaderElection.ResourceNamespace,
LeaderElection: config.LeaderElection.LeaderElect,
LeaderElectionID: config.LeaderElection.ResourceName,
LeaderElectionNamespace: config.LeaderElection.ResourceNamespace,
})
if err != nil {
return fmt.Errorf("failed to build controller manager: %v", err)
}

hostKubeClient, err := kubernetes.NewForConfig(config)
hostKubeClient, err := kubernetes.NewForConfig(config.RestConfig)
if err != nil {
return fmt.Errorf("could not create clientset: %v", err)
}

kosmosClient, err := versioned.NewForConfig(config)
kosmosClient, err := versioned.NewForConfig(config.RestConfig)
if err != nil {
return fmt.Errorf("could not create clientset: %v", err)
}
@@ -127,7 +248,7 @@ func run(ctx context.Context, opts *options.Options) error {
EventRecorder: mgr.GetEventRecorderFor(constants.InitControllerName),
RootClientSet: hostKubeClient,
KosmosClient: kosmosClient,
KubeNestOptions: &opts.KubeNestOptions,
KubeNestOptions: &config.KubeNestOptions,
}
if err = VirtualClusterInitController.SetupWithManager(mgr); err != nil {
return fmt.Errorf("error starting %s: %v", constants.InitControllerName, err)
@@ -153,19 +274,20 @@ func run(ctx context.Context, opts *options.Options) error {
hostKubeClient,
mgr.GetEventRecorderFor(constants.NodeControllerName),
kosmosClient,
&opts.KubeNestOptions,
&config.KubeNestOptions,
)

if err = VirtualClusterNodeController.SetupWithManager(mgr); err != nil {
return fmt.Errorf("error starting %s: %v", constants.NodeControllerName, err)
}

if opts.KosmosJoinController {
if config.KubeNestOptions.KubeNestType == v1alpha1.KosmosKube {
KosmosJoinController := kosmos.KosmosJoinController{
Client: mgr.GetClient(),
EventRecorder: mgr.GetEventRecorderFor(constants.KosmosJoinControllerName),
KubeconfigPath: opts.KubernetesOptions.KubeConfig,
AllowNodeOwnbyMulticluster: opts.AllowNodeOwnbyMulticluster,
KubeConfig: config.RestConfig,
KubeconfigStream: config.KubeconfigStream,
AllowNodeOwnbyMulticluster: config.KubeNestOptions.KosmosKubeConfig.AllowNodeOwnbyMulticluster,
}
if err = KosmosJoinController.SetupWithManager(mgr); err != nil {
return fmt.Errorf("error starting %s: %v", constants.KosmosJoinControllerName, err)
21 changes: 13 additions & 8 deletions cmd/kubenest/operator/app/options/options.go
Original file line number Diff line number Diff line change
@@ -5,15 +5,19 @@ import (
"k8s.io/client-go/tools/leaderelection/resourcelock"
componentbaseconfig "k8s.io/component-base/config"

"github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1"
"github.com/kosmos.io/kosmos/pkg/utils"
)

type Options struct {
LeaderElection componentbaseconfig.LeaderElectionConfiguration
KubernetesOptions KubernetesOptions
KubeNestOptions KubeNestOptions
DeprecatedOptions v1alpha1.KubeNestConfiguration
AllowNodeOwnbyMulticluster bool
KosmosJoinController bool

// ConfigFile is the location of the kubenest's configuration file.
ConfigFile string
}

type KubernetesOptions struct {
@@ -58,11 +62,12 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) {
flags.StringVar(&o.KubernetesOptions.Master, "master", "", "Used to generate kubeconfig for downloading, if not specified, will use host in kubeconfig.")
flags.BoolVar(&o.AllowNodeOwnbyMulticluster, "multiowner", false, "Allow node own by multicluster or not.")
flags.BoolVar(&o.KosmosJoinController, "kosmos-join-controller", false, "Turn on or off kosmos-join-controller.")
flags.BoolVar(&o.KubeNestOptions.ForceDestroy, "kube-nest-force-destroy", false, "Force destroy the node.If it set true.If set to true, Kubernetes will not evict the existing nodes on the node when joining nodes to the tenant's control plane, but will instead force destroy.")
flags.StringVar(&o.KubeNestOptions.AnpMode, "kube-nest-anp-mode", "tcp", "kube-apiserver network proxy mode, must be set to tcp or uds. uds mode the replicas for apiserver should be one, and tcp for multi apiserver replicas.")
flags.BoolVar(&o.KubeNestOptions.AdmissionPlugins, "kube-nest-admission-plugins", false, "kube-apiserver network disable-admission-plugins, false for - --disable-admission-plugins=License, true for remove the --disable-admission-plugins=License flag .")
flags.IntVar(&o.KubeNestOptions.ApiServerReplicas, "kube-nest-apiserver-replicas", 1, "virtual-cluster kube-apiserver replicas. default is 2.")
flags.StringVar(&o.KubeNestOptions.ClusterCIDR, "cluster-cidr", "10.244.0.0/16", "Used to set the cluster-cidr of kube-controller-manager and kube-proxy (configmap)")
flags.StringVar(&o.KubeNestOptions.ETCDStorageClass, "etcd-storage-class", "openebs-hostpath", "Used to set the etcd storage class.")
flags.StringVar(&o.KubeNestOptions.ETCDUnitSize, "etcd-unit-size", "1Gi", "Used to set the etcd unit size, each node is allocated storage of etcd-unit-size.")
flags.BoolVar(&o.DeprecatedOptions.KubeInKubeConfig.ForceDestroy, "kube-nest-force-destroy", false, "Force destroy the node.If it set true.If set to true, Kubernetes will not evict the existing nodes on the node when joining nodes to the tenant's control plane, but will instead force destroy.")
flags.StringVar(&o.DeprecatedOptions.KubeInKubeConfig.AnpMode, "kube-nest-anp-mode", "tcp", "kube-apiserver network proxy mode, must be set to tcp or uds. uds mode the replicas for apiserver should be one, and tcp for multi apiserver replicas.")
flags.BoolVar(&o.DeprecatedOptions.KubeInKubeConfig.AdmissionPlugins, "kube-nest-admission-plugins", false, "kube-apiserver network disable-admission-plugins, false for - --disable-admission-plugins=License, true for remove the --disable-admission-plugins=License flag .")
flags.IntVar(&o.DeprecatedOptions.KubeInKubeConfig.ApiServerReplicas, "kube-nest-apiserver-replicas", 1, "virtual-cluster kube-apiserver replicas. default is 2.")
flags.StringVar(&o.DeprecatedOptions.KubeInKubeConfig.ClusterCIDR, "cluster-cidr", "10.244.0.0/16", "Used to set the cluster-cidr of kube-controller-manager and kube-proxy (configmap)")
flags.StringVar(&o.DeprecatedOptions.KubeInKubeConfig.ETCDStorageClass, "etcd-storage-class", "openebs-hostpath", "Used to set the etcd storage class.")
flags.StringVar(&o.DeprecatedOptions.KubeInKubeConfig.ETCDUnitSize, "etcd-unit-size", "1Gi", "Used to set the etcd unit size, each node is allocated storage of etcd-unit-size.")
flags.StringVar(&o.ConfigFile, "config", "", "The path to the configuration file.")
}
10 changes: 0 additions & 10 deletions cmd/kubenest/operator/app/options/validation.go

This file was deleted.

65 changes: 65 additions & 0 deletions pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package v1alpha1

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type KubeNestType string

const (
KubeInKube KubeNestType = "Kube in kube"
KosmosKube KubeNestType = "Kosmos in kube"
)

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// KubeNestConfiguration defines the configuration for KubeNest
type KubeNestConfiguration struct {
// TypeMeta contains the API version and kind.
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

KubeNestType KubeNestType `yaml:"kubeNestType" json:"kubeNestType,omitempty"`

KosmosKubeConfig KosmosKubeConfig `yaml:"kosmosKubeConfig" json:"kosmosKubeConfig,omitempty"`

KubeInKubeConfig KubeInKubeConfig `yaml:"kubeInKubeConfig" json:"kubeInKubeConfig,omitempty"`
}

type EtcdCluster struct {
}

type KosmosKubeConfig struct {
// AllowNodeOwnbyMulticluster indicates whether to allow nodes to be owned by multiple clusters.
AllowNodeOwnbyMulticluster bool `yaml:"allowNodeOwnbyMulticluster" json:"allowNodeOwnbyMulticluster,omitempty"`
}

type KubeInKubeConfig struct {
// todo Group according to the parameters of apiserver, etcd, coredns, etc.

ForceDestroy bool `yaml:"forceDestroy" json:"forceDestroy,omitempty"`
AnpMode string `yaml:"anpMode" json:"anpMode,omitempty"`
AdmissionPlugins bool `yaml:"admissionPlugins" json:"admissionPlugins,omitempty"`
ApiServerReplicas int `yaml:"apiServerReplicas" json:"apiServerReplicas,omitempty"`
ClusterCIDR string `yaml:"clusterCIDR" json:"clusterCIDR,omitempty"`
ETCDStorageClass string `yaml:"etcdStorageClass" json:"etcdStorageClass,omitempty"`
ETCDUnitSize string `yaml:"etcdUnitSize" json:"etcdUnitSize,omitempty"`

//// Etcd contains the configuration for the etcd statefulset.
//Etcd EtcdCluster `yaml:"etcd" json:"etcd,omitempty"`

//// DNS contains the configuration for the dns server in kubernetes.
//DNS DNS `yaml:"dns" json:"dns,omitempty"`
//
//// Kubernetes contains the configuration for the kubernetes.
//Kubernetes Kubernetes `yaml:"kubernetes" json:"kubernetes,omitempty"`
//
//// Network contains the configuration for the network in kubernetes cluster.
//Network NetworkConfig `yaml:"network" json:"network,omitempty"`
//
//// Storage contains the configuration for the storage in kubernetes cluster.
//Storage StorageConfig `yaml:"storage" json:"storage,omitempty"`
//
//// Registry contains the configuration for the registry in kubernetes cluster.
//Registry RegistryConfig `yaml:"registry" json:"registry,omitempty"`
}
Loading

0 comments on commit c65778d

Please sign in to comment.