feat: Apiserver supports exposing virtual cluster access using nodePort mode

.gitignore

@@ -34,4 +34,6 @@ ignore_dir
 cmd/kubenest/node-agent/app.log
 cmd/kubenest/node-agent/cert.pem
 cmd/kubenest/node-agent/key.pem
-cmd/kubenest/node-agent/agent.env
+cmd/kubenest/node-agent/agent.env
+hack/k8s-in-k8s/nodes.txt
+develop

deploy/crds/kosmos.io_kubenestconfigurations.yaml

@@ -45,6 +45,12 @@ spec:
                 type: string
               apiServerReplicas:
                 type: integer
+              apiServerServiceType:
+                default: hostNetwork
+                enum:
+                - nodePort
+                - hostNetwork
+                type: string
               clusterCIDR:
                 type: string
               etcdStorageClass:

deploy/crds/kosmos.io_virtualclusters.yaml

@@ -62,6 +62,12 @@ spec:
                     type: string
                   apiServerReplicas:
                     type: integer
+                  apiServerServiceType:
+                    default: hostNetwork
+                    enum:
+                    - nodePort
+                    - hostNetwork
+                    type: string
                   clusterCIDR:
                     type: string
                   etcdStorageClass:

deploy/virtual-cluster-operator.yml

@@ -252,45 +252,3 @@ data:
   username: {{ .USERNAME }}
   password: {{ .PASSWORD }}
 ---
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kosmos-hostports
-  namespace: kosmos-system
-data:
-  config.yaml: |
-    # ports allocate for virtual cluster api server,from 33001, increment by 1 for each virtual cluster.Be careful not to use ports that are already in use
-    portsPool:
-      - 33001
-      - 33002
-      - 33003
-      - 33004
-      - 33005
-      - 33006
-      - 33007
-      - 33008
-      - 33009
-      - 33010
----
-apiVersion: v1
-data:
-  egress_selector_configuration.yaml: |
-    apiVersion: apiserver.k8s.io/v1beta1
-    kind: EgressSelectorConfiguration
-    egressSelections:
-      - name: cluster
-        connection:
-          proxyProtocol: GRPC
-          transport:
-            uds:
-              udsName: /etc/kubernetes/konnectivity-server/konnectivity-server.socket
-      - name: master
-        connection:
-          proxyProtocol: Direct
-      - name: etcd
-        connection:
-          proxyProtocol: Direct
-kind: ConfigMap
-metadata:
-  name: kas-proxy-files
-  namespace: kas-proxy

hack/k8s-in-k8s/g.env.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+# #####
+# Generate by script generate_env.sh
+# #####
+
+SCRIPT_VERSION=0.0.1
+# tmp dir of kosmos
+PATH_FILE_TMP=/apps/conf/kosmos/tmp
+##################################################
+# path for kubeadm config
+PATH_KUBEADM_CONFIG=/etc/kubeadm
+##################################################
+# path for kubernetes, from kubelet args --config
+PATH_KUBERNETES=/etc/kubernetes
+PATH_KUBERNETES_PKI=/etc/kubernetes/pki
+# name for kubelet kubeconfig file
+KUBELET_KUBE_CONFIG_NAME=
+##################################################
+# path for kubelet
+PATH_KUBELET_LIB=/var/lib/kubelet
+# path for kubelet 
+PATH_KUBELET_CONF=.
+# name for config file of kubelet
+KUBELET_CONFIG_NAME=
+HOST_CORE_DNS=10.96.0.10
+# kubeadm switch
+USE_KUBEADM=true
+# Generate kubelet.conf TIMEOUT
+KUBELET_CONF_TIMEOUT=30
+
+function GenerateKubeadmConfig() {
+    echo "---
+apiVersion: kubeadm.k8s.io/v1beta2
+caCertPath: /etc/kubernetes/pki/ca.crt
+discovery:
+    bootstrapToken:
+        apiServerEndpoint: apiserver.cluster.local:6443
+        token: $1
+        unsafeSkipCAVerification: true
+kind: JoinConfiguration
+nodeRegistration:
+    criSocket: /run/containerd/containerd.sock
+    kubeletExtraArgs:
+    container-runtime: remote
+    container-runtime-endpoint: unix:///run/containerd/containerd.sock
+    taints: null" > $2/kubeadm.cfg.current
+}
+
+function GenerateStaticNginxProxy() {
+    echo "apiVersion: v1
+kind: Pod
+metadata:
+  creationTimestamp: null
+  name: nginx-proxy
+  namespace: kube-system
+spec:
+  containers:
+  - image: registry.paas/cmss/nginx:1.21.4
+    imagePullPolicy: IfNotPresent
+    name: nginx-proxy
+    resources:
+      limits:
+        cpu: 300m
+        memory: 512M
+      requests:
+        cpu: 25m
+        memory: 32M
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /etc/nginx
+      name: etc-nginx
+      readOnly: true
+  hostNetwork: true
+  priorityClassName: system-node-critical
+  volumes:
+  - hostPath:
+      path: /apps/conf/nginx
+      type: 
+    name: etc-nginx
+status: {}" > /etc/kubernetes/manifests/nginx-proxy.yaml
+}
+
+

hack/k8s-in-k8s/generate_env.sh

@@ -124,7 +124,7 @@ PATH_KUBELET_CONF=$PATH_KUBELET_CONF
 KUBELET_CONFIG_NAME=$KUBELET_CONFIG_NAME
 HOST_CORE_DNS=$HOST_CORE_DNS
 # kubeadm switch
-USE_KUBEADM=true
+USE_KUBEADM=false
 # Generate kubelet.conf TIMEOUT
 KUBELET_CONF_TIMEOUT=30
 

pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go

@@ -11,6 +11,13 @@ const (
 	KosmosKube KubeNestType = "Kosmos in kube"
 )
 
+type ApiServerServiceType string
+
+const (
+	HostNetwork ApiServerServiceType = "hostNetwork"
+	NodePort    ApiServerServiceType = "nodePort"
+)
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // KubeNestConfiguration defines the configuration for KubeNest
@@ -73,6 +80,11 @@ type KubeInKubeConfig struct {
 	//TenantEntrypoint TenantEntrypoint `yaml:"tenantEntrypoint" json:"tenantEntrypoint,omitempty"`
 	// +optional
 	TenantEntrypoint TenantEntrypoint `yaml:"tenantEntrypoint" json:"tenantEntrypoint,omitempty"`
+
+	// +kubebuilder:validation:Enum=nodePort;hostNetwork
+	// +kubebuilder:default=hostNetwork
+	// +optional
+	ApiServerServiceType ApiServerServiceType `yaml:"apiServerServiceType" json:"apiServerServiceType,omitempty"`
 }
 
 // TenantEntrypoint contains the configuration for the tenant entrypoint.

pkg/apis/kosmos/v1alpha1/virtualcluster_types.go

@@ -58,7 +58,7 @@ type VirtualClusterSpec struct {
 
 	// KubeInKubeConfig is the external config of virtual cluster
 	// +optional
-	KubeInKubeConfig KubeInKubeConfig `json:"kubeInKubeConfig,omitempty"`
+	KubeInKubeConfig *KubeInKubeConfig `json:"kubeInKubeConfig,omitempty"`
 	// PromotePolicies definites the policies for promote to the kubernetes's control plane
 	// +required
 	PromotePolicies []PromotePolicy `json:"promotePolicies,omitempty"`