Skip to content

Commit

Permalink
Merge pull request kosmos-io#466 from yuleichun-striving/main
Browse files Browse the repository at this point in the history
add codes for the creating of virtual cluster for tenants
  • Loading branch information
kosmos-robot authored Apr 15, 2024
2 parents 423aed5 + 8e15c08 commit b297300
Show file tree
Hide file tree
Showing 54 changed files with 3,498 additions and 151 deletions.
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ MACOS_TARGETS := clusterlink-controller-manager \
clusterlink-network-manager \
clusterlink-proxy \
clustertree-cluster-manager \
virtual-cluster-operator \
scheduler \

# clusterlink-agent and clusterlink-floater only support linux platform
Expand All @@ -26,6 +27,7 @@ TARGETS := clusterlink-controller-manager \
clusterlink-network-manager \
clusterlink-proxy \
clustertree-cluster-manager \
virtual-cluster-operator \
scheduler \

# If GOOS is macOS, assign the value of MACOS_TARGETS to TARGETS
Expand Down Expand Up @@ -123,6 +125,7 @@ upload-images: images
docker push ${REGISTRY}/clusterlink-floater:${VERSION}
docker push ${REGISTRY}/clusterlink-elector:${VERSION}
docker push ${REGISTRY}/clustertree-cluster-manager:${VERSION}
docker push ${REGISTRY}/virtual-cluster-operator:${VERSION}
docker push ${REGISTRY}/scheduler:${VERSION}

.PHONY: release
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@ import (
"k8s.io/klog/v2"
controllerruntime "sigs.k8s.io/controller-runtime"

"github.com/kosmos.io/kosmos/cmd/clustertree-operator/app/options"
"github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/options"
"github.com/kosmos.io/kosmos/pkg/kubenest/constants"
"github.com/kosmos.io/kosmos/pkg/kubenest/controller"
kosmos "github.com/kosmos.io/kosmos/pkg/kubenest/controller/kosmos"
"github.com/kosmos.io/kosmos/pkg/scheme"
"github.com/kosmos.io/kosmos/pkg/sharedcli/klogflag"
"github.com/kosmos.io/kosmos/pkg/treeoperator/constants"
"github.com/kosmos.io/kosmos/pkg/treeoperator/controller"
)

func NewVirtualClusterOperatorCommand(ctx context.Context) *cobra.Command {
Expand Down Expand Up @@ -82,14 +83,16 @@ func run(ctx context.Context, opts *options.Options) error {
return fmt.Errorf("error starting %s: %v", constants.InitControllerName, err)
}

VirtualClusterJoinController := controller.VirtualClusterJoinController{
Client: mgr.GetClient(),
EventRecorder: mgr.GetEventRecorderFor(constants.JoinControllerName),
KubeconfigPath: opts.KubernetesOptions.KubeConfig,
AllowNodeOwnbyMulticluster: opts.AllowNodeOwnbyMulticluster,
}
if err = VirtualClusterJoinController.SetupWithManager(mgr); err != nil {
return fmt.Errorf("error starting %s: %v", constants.JoinControllerName, err)
if opts.KosmosJoinController {
KosmosJoinController := kosmos.KosmosJoinController{
Client: mgr.GetClient(),
EventRecorder: mgr.GetEventRecorderFor(constants.KosmosJoinControllerName),
KubeconfigPath: opts.KubernetesOptions.KubeConfig,
AllowNodeOwnbyMulticluster: opts.AllowNodeOwnbyMulticluster,
}
if err = KosmosJoinController.SetupWithManager(mgr); err != nil {
return fmt.Errorf("error starting %s: %v", constants.KosmosJoinControllerName, err)
}
}

if err := mgr.Start(ctx); err != nil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ type Options struct {
LeaderElection componentbaseconfig.LeaderElectionConfiguration
KubernetesOptions KubernetesOptions
AllowNodeOwnbyMulticluster bool
KosmosJoinController bool
}

type KubernetesOptions struct {
Expand All @@ -38,11 +39,12 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) {
}

flags.BoolVar(&o.LeaderElection.LeaderElect, "leader-elect", true, "Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.")
flags.StringVar(&o.LeaderElection.ResourceName, "leader-elect-resource-name", "clustertree-operator", "The name of resource object that is used for locking during leader election.")
flags.StringVar(&o.LeaderElection.ResourceName, "leader-elect-resource-name", "operator", "The name of resource object that is used for locking during leader election.")
flags.StringVar(&o.LeaderElection.ResourceNamespace, "leader-elect-resource-namespace", utils.DefaultNamespace, "The namespace of resource object that is used for locking during leader election.")
flags.Float32Var(&o.KubernetesOptions.QPS, "kube-qps", 40.0, "QPS to use while talking with kube-apiserver.")
flags.IntVar(&o.KubernetesOptions.Burst, "kube-burst", 60, "Burst to use while talking with kube-apiserver.")
flags.StringVar(&o.KubernetesOptions.KubeConfig, "kubeconfig", "", "Path for kubernetes kubeconfig file, if left blank, will use in cluster way.")
flags.StringVar(&o.KubernetesOptions.Master, "master", "", "Used to generate kubeconfig for downloading, if not specified, will use host in kubeconfig.")
flags.BoolVar(&o.AllowNodeOwnbyMulticluster, "multiowner", false, "Allow node own by multicluster or not.")
flags.BoolVar(&o.KosmosJoinController, "kosmos-join-controller", false, "Turn on or off kosmos-join-controller.")
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"k8s.io/component-base/cli"
ctrl "sigs.k8s.io/controller-runtime"

"github.com/kosmos.io/kosmos/cmd/clustertree-operator/app"
"github.com/kosmos.io/kosmos/cmd/kubenest/operator/app"
)

func main() {
Expand Down
24 changes: 19 additions & 5 deletions deploy/crds/kosmos.io_virtualclusters.yaml
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@

---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.11.0
controller-gen.kubebuilder.io/version: v0.7.0
creationTimestamp: null
name: virtualclusters.kosmos.io
spec:
Expand Down Expand Up @@ -43,11 +44,18 @@ spec:
to the kubernetes's control plane, the resources can be nodes or
just cpu,memory or gpu resources
properties:
nodes:
description: Nodes is the names of node to promote to the kubernetes's
control plane
nodeInfos:
description: NodeInfos is the info of nodes to promote to the
kubernetes's control plane
items:
type: string
properties:
address:
description: Address defines node ip
type: string
nodeName:
description: NodeName defines node name
type: string
type: object
type: array
resources:
additionalProperties:
Expand Down Expand Up @@ -75,3 +83,9 @@ spec:
type: object
served: true
storage: true
status:
acceptedNames:
kind: ""
plural: ""
conditions: []
storedVersions: []
77 changes: 77 additions & 0 deletions deploy/virtual-cluster-operator.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: virtual-cluster-operator
namespace: kosmos-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: virtual-cluster-operator
rules:
- apiGroups: ['*']
resources: ['*']
verbs: ["*"]
- nonResourceURLs: ['*']
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: virtual-cluster-operator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: virtual-cluster-operator
subjects:
- kind: ServiceAccount
name: virtual-cluster-operator
namespace: kosmos-system
---
apiVersion: v1
kind: Secret
metadata:
name: virtual-cluster-operator
namespace: kosmos-system
type: Opaque
data:
kubeconfig: __kubeconfig__

---
apiVersion: apps/v1
kind: Deployment
metadata:
name: virtual-cluster-operator
namespace: kosmos-system
labels:
app: virtual-cluster-operator
spec:
replicas: 1
selector:
matchLabels:
app: virtual-cluster-operator
template:
metadata:
labels:
app: virtual-cluster-operator
spec:
serviceAccountName: virtual-cluster-operator
containers:
- name: virtual-cluster-operator
image: repo1-cn-beijing.cr.volces.com/virtualcluster/virtualcluter/virtual-cluster-operator:ee1bcd33-dirty
imagePullPolicy: IfNotPresent
env:
- name: IMAGE_REPOSITIRY
value: repo1-cn-beijing.cr.volces.com/virtualcluster/virtualcluter
volumeMounts:
- name: credentials
mountPath: /etc/virtual-cluster-operator
readOnly: true
command:
- virtual-cluster-operator
- --kubeconfig=/etc/virtual-cluster-operator/kubeconfig
- --v=4
volumes:
- name: credentials
secret:
secretName: virtual-cluster-operator
14 changes: 9 additions & 5 deletions examples/vitual-cluster-demo.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@ metadata:
name: test
spec:
promoteResources:
nodes:
- node1
- node2
nodeInfos:
- nodeName: nodeName1
address: 127.0.0.1
- nodeName: nodeName2
address: 127.0.0.2
resources:
cpu: 1
memory: "1Gi"
cpu: 2
memory: "1Gi"
status:
phase: Completed
1 change: 1 addition & 0 deletions hack/util.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ CLUSTERLINK_TARGET_SOURCE=(
clusterlink-network-manager=cmd/clusterlink/network-manager
clusterlink-controller-manager=cmd/clusterlink/controller-manager
clustertree-cluster-manager=cmd/clustertree/cluster-manager
virtual-cluster-operator=cmd/kubenest/operator
kosmosctl=cmd/kosmosctl
)

Expand Down
13 changes: 11 additions & 2 deletions pkg/apis/kosmos/v1alpha1/virtualcluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,24 @@ type VirtualClusterSpec struct {
}

type PromoteResources struct {
// Nodes is the names of node to promote to the kubernetes's control plane
// NodeInfos is the info of nodes to promote to the kubernetes's control plane
// +optional
Nodes []string `json:"nodes,omitempty"`
NodeInfos []NodeInfo `json:"nodeInfos,omitempty"`

// Resources is the resources to promote to the kubernetes's control plane
// +optional
Resources corev1.ResourceList `json:"resources,omitempty"`
}

type NodeInfo struct {
//NodeName defines node name
//+optional
NodeName string `json:"nodeName,omitempty"`
//Address defines node ip
//+optional
Address string `json:"address,omitempty"`
}

type VirtualClusterStatus struct {
// Phase is the phase of kosmos-operator handling the VirtualCluster
// +optional
Expand Down
6 changes: 3 additions & 3 deletions pkg/apis/kosmos/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

80 changes: 80 additions & 0 deletions pkg/kubenest/constants/constant.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package constants

import "time"

const (
InitControllerName = "virtual-cluster-init-controller"
KosmosJoinControllerName = "kosmos-join-controller"
SystemNs = "kube-system"
DefauleImageRepositoryEnv = "IMAGE_REPOSITIRY"
DefauleImageVersionEnv = "IMAGE_VERSION"
VirtualClusterStatusCompleted = "Completed"
VirtualClusterFinalizerName = "kosmos.io/virtual-cluster-finalizer"
ServiceType = "NodePort"
EtcdServiceType = "ClusterIP"
DisableCascadingDeletionLabel = "operator.virtualcluster.io/disable-cascading-deletion"
ControllerFinalizerName = "operator.virtualcluster.io/finalizer"
DefaultKubeconfigPath = "/etc/cluster-tree/cert"
Label = "virtualCluster-app"
ComponentBeReadyTimeout = 120 * time.Second

// CertificateBlockType is a possible value for pem.Block.Type.
CertificateBlockType = "CERTIFICATE"
RsaKeySize = 2048
KeyExtension = ".key"
CertExtension = ".crt"
CertificateValidity = time.Hour * 24 * 365
CaCertAndKeyName = "ca"
VirtualClusterCertAndKeyName = "virtualCluster"
VirtualClusterSystemNamespace = "virtualCluster-system"
ApiserverCertAndKeyName = "apiserver"
EtcdCaCertAndKeyName = "etcd-ca"
EtcdServerCertAndKeyName = "etcd-server"
EtcdClientCertAndKeyName = "etcd-client"
FrontProxyCaCertAndKeyName = "front-proxy-ca"
FrontProxyClientCertAndKeyName = "front-proxy-client"

//controlplane apiserver
ApiServer = "apiserver"
ApiServerReplicas = 1
ApiServerServiceSubnet = "10.237.6.18/29"
ApiServerEtcdListenClientPort = 2379
ApiServerServiceType = "NodePort"
// APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation
ApiServerCallRetryInterval = 100 * time.Millisecond

//controlplane etcd
Etcd = "etcd"
EtcdReplicas = 3
EtcdDataVolumeName = "etcd-data"
EtcdListenClientPort = 2379
EtcdListenPeerPort = 2380

//controlplane kube-controller
KubeControllerReplicas = 1
KubeControllerManagerComponent = "KubeControllerManager"
KubeControllerManager = "kube-controller-manager"

//controlplane scheduler
VirtualClusterSchedulerReplicas = 1
VirtualClusterSchedulerComponent = "VirtualClusterScheduler"
VirtualClusterSchedulerComponentConfigMap = "scheduler-config"
VirtualClusterScheduler = "virtualCluster-scheduler"

//controlplane auth
AdminConfig = "admin-config"
KubeConfig = "kubeconfig"

//controlplane upload
VirtualClusterLabelKeyName = "app.kubernetes.io/managed-by"
VirtualClusterController = "virtual-cluster-controller"
ClusterName = "virtualCluster-apiserver"
UserName = "virtualCluster-admin"

// InitAction represents init virtual cluster instance
InitAction Action = "init"
// DeInitAction represents delete virtual cluster instance
DeInitAction Action = "deInit"
)

type Action string
Loading

0 comments on commit b297300

Please sign in to comment.