Merge branch 'kosmos-io:main' into fix_clustertree

deploy/clustertree-cluster-manager.yml

@@ -35,8 +35,8 @@ metadata:
   namespace: kosmos-system
 type: Opaque
 data:
-  cert.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQzakNDQXNhZ0F3SUJBZ0lJVWE0NWVxZmI0c0V3RFFZSktvWklodmNOQVFFTEJRQXdmekVMTUFrR0ExVUUKQmhNQ1ZWTXhEekFOQmdOVkJBZ1RCazl5WldkdmJqRVJNQThHQTFVRUJ4TUlVRzl5ZEd4aGJtUXhHREFXQmdOVgpCQW9URDNacmRXSmxiR1YwTFcxdlkyc3RNREVZTUJZR0ExVUVDeE1QZG10MVltVnNaWFF0Ylc5amF5MHdNUmd3CkZnWURWUVFERXc5MmEzVmlaV3hsZEMxdGIyTnJMVEF3SGhjTk1UZ3hNVEkyTVRJd016SXpXaGNOTVRrd01qSTEKTVRnd09ESXpXakIvTVFzd0NRWURWUVFHRXdKVlV6RVBNQTBHQTFVRUNCTUdUM0psWjI5dU1SRXdEd1lEVlFRSApFd2hRYjNKMGJHRnVaREVZTUJZR0ExVUVDaE1QZG10MVltVnNaWFF0Ylc5amF5MHdNUmd3RmdZRFZRUUxFdzkyCmEzVmlaV3hsZEMxdGIyTnJMVEF4R0RBV0JnTlZCQU1URDNacmRXSmxiR1YwTFcxdlkyc3RNRENDQVNJd0RRWUoKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTHJ5SHZLM1VCQkJxR1YyRnB3eW1mMHAvWUtHUUE5cgpOdTBONmYyK1JrVVhMdVFYRytXZEZRbDNaUXliUExmQ0UyaHdGY2wzSUYrM2hDelkzLzJVSXlHQmxvQklmdDdLCllGTE0zWVdKRHk1RWxLRGcxYk5EU0x6RjZ0a3BOTERuVmxna1BQSVR6cEVISUF1K0JUNURaR1doWUFXTy9EaXIKWGR4b0pCT2hQWlpDY0JDVitrd1FRUGJzWHpaeStxN1FoeDI3MENSTUlYc285QzVMSmhHWUw5ZndzeG11a0FPUgo1NlNtZnNBYW1sN1VPbHpISVRSRHdENUFRMUJrVFNFRnkwOGRrNkpBWUw4TERMaGdhTG9Xb1YwR2UyZ09JZXBSCmpwbDg3ZEdiU1ZHeUJIbVRYdjRvNnV0cVQ2UzZuVTc2TG45TlNpN1loTXFqOHVXdjBwVERsWWNDQXdFQUFhTmUKTUZ3d0RnWURWUjBQQVFIL0JBUURBZ1dnTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjRApBakFNQmdOVkhSTUJBZjhFQWpBQU1CMEdBMVVkRGdRV0JCUVZId1Uxc3k3UW53MVd2VnZGTGNacmhvVDQwREFOCkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQXNOR05LejFKd2Z3ZzdyWWFPN1ZGL3phbjAxWFhaRlAxYm5GWW5YSnUKMTVSemhPQk1zcDNLdldDVmh3VWZ4TmU4R2hVRFN4MnRtUzVFQS84b2FFbmdMRmwzanRSM3BuVU5Pd0RWbHpseQpRT0NOM3JsT2k0K3AyNkx2TWlBRnA1aHhYQXYzTE9SczZEenI2aDMvUVR0bFY1akRTaFVPWFpkRmRPUEpkWjJuCmc0Ymlyckc3TU82dnd2UjhDaU5jUTI2YitiOHA5QkdYYkU4YnNKb0htY3NxeWE4ZmJWczJuNkNkRUplSSs0aEQKTjZ4bG81U3Zoakg1dEZJSTdlQ1ZlZHlaR2wwQkt2a29jT2lnTGdxOFgrSnpGeGoxd3RkbXRYdjdzamRLY0I5cgo2VFdHSlJyWlZ4b3hVT3paaHB4VWozai9wTGFSY0RtdHRTSkN1RHUzTkF0a2dRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
-  key.pem: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdXZJZThyZFFFRUdvWlhZV25ES1ovU245Z29aQUQyczI3UTNwL2I1R1JSY3U1QmNiCjVaMFZDWGRsREpzOHQ4SVRhSEFWeVhjZ1g3ZUVMTmpmL1pRaklZR1dnRWgrM3NwZ1VzemRoWWtQTGtTVW9PRFYKczBOSXZNWHEyU2swc09kV1dDUTg4aFBPa1FjZ0M3NEZQa05rWmFGZ0JZNzhPS3RkM0dna0U2RTlsa0p3RUpYNgpUQkJBOXV4Zk5uTDZydENISGJ2UUpFd2hleWowTGtzbUVaZ3YxL0N6R2E2UUE1SG5wS1ord0JxYVh0UTZYTWNoCk5FUEFQa0JEVUdSTklRWExUeDJUb2tCZ3Z3c011R0JvdWhhaFhRWjdhQTRoNmxHT21YenQwWnRKVWJJRWVaTmUKL2lqcTYycFBwTHFkVHZvdWYwMUtMdGlFeXFQeTVhL1NsTU9WaHdJREFRQUJBb0lCQUVOODR0VkdmaDNRUmlXUwpzdWppajVySVROK1E3WkZqYUNtOTZ5b1NSYlh0ZjUwU0JwMG16eEJpek5UM09iMHd6K2JWQjloNksvTENBbkphClBNcURid2RLaS9WMXRtOWhhZEthYUtJcmI1S0phWXFHZ0Q4OTNBVmlBYjB4MWZiREhQV201MldRNXZLT092QmkKUWV4UFVmQXFpTXFZNnM3ZWRuejZENFFTb25RYW14Q1VQQlBZdnVkbWF5SHRQbGM4UWI2ZVkwVitwY2RGblcwOApTRFpYWU94ZXkzL0lBalp5ZGNBN1hndk5TYys2WE93bWhLc0dBVzcxdUZUVGFnSnZ6WDNlUENZMTRya0dKbURHCm0vMTBob1c2Tk1LR2VWL1J5WDNkWDBqSm1EazFWZnhBUVczeHBPaXBaZmdmdmdhdkNPcUhuS0E2SThkSzN6aGcKdkU5QmxlRUNnWUVBODdYL3p0UVpESTRxT1RBOUNXL25NWGZ3QXk5UU8xSzZiR2hCSFV1N0pzNHBxZ3h1SDhGawpoUWdRSzdWOGlhc255L2RDeWo2Q3UzUUpOb2Z4dWRBdkxMUUtrcXV5UU9hK3pxRkNVcFZpZDdKVlJNY1JMSmx0CjNIbHlDTnZWbGhmakRUMGNJMlJkVTQ1cThNblpveTFmM0RQWkIxNmNIYjNITDl6MWdRWlRpWEVDZ1lFQXhGOWEKNjhTYnhtV0ZCSzdQYW9iSTh3VmZEb1Rpckhtb0F2bnlwWUswb1FrQVg4Vm1FbXRFRXMyK04xbW9LalNUUHIrdAp1czRKS2d1QTh6MnR1TGs1aitlRit6RGwvMlUrN2RqVEY4RkNOcHJ3ejNzWHI0MjdHQ0lHTDVZdnBJQlorVEw4CkJqaTJ1eW9vOGs5U0FXTWI0T2JPemZHbTR0ZUN2Y2lTOTlndzBuY0NnWUF0NUdiQVZ0WkVzL3lsZWp6MEt2dFoKS0dHczU5cnU0TncwRDhtN0w0aVZmUnNCWjRmUk9RU3B2R1AzSnh6RmU5SnBxUzBOa29uaHJLOFRjclFGTG52RApxaitYY1BlSEd5eHhFcEsvcEZ1L2VIaHdGQ0JheXFXU2I5Z1diUGNpWldzZkVoUGJZa25rc3h2V0xkeHF5dCtUClFyd3FsQmxIekhYV3dJQUdoTjkwTVFLQmdRQzVDWWtwQkZnc3VGaUJNeCtySjFxTzlJNi9wYVBhRmNDbEhWVHgKZEpvejY4RjRmUTlUWjlQN1MvZGpQSTVqUnF0QXcyazJ6eEovbGR0cVdNSXJnQTJuZGVnZjY5R3R1SDkxcTR3dApwQ042Uk1HSklGb1BTQ1AxOTRtUXFabzNEZUs2R0xxMk9oYWxnbktXOFBzNjUyTExwM0ZUU2RPUmlMVmZrM0k1CkxIUEV2UUtCZ0RDeGEvM3ZuZUc4dmdzOEFyRWpOODlCL1l4TzFxSVU1bXhKZTZaYWZiODFOZGhZVWpmUkFWcm8KQUxUb2ZpQXBNc25EYkpESE1pd3Z3Y0RVSGJQTHBydUs4MFIvL3ptWDdYZW4rRis1b2JmU1E4ajBHU21tZVdGUQpTVkc2QXBOdGt0TFBJMG5LMm5FSUgvUXg0b3VHQzlOMHBBRFJDbFFRUFN4RVBtRHZmNHhmCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCgo=
+  cert.pem: __CERT__
+  key.pem: __KEY__
 
 ---
 apiVersion: apps/v1
@@ -66,10 +66,12 @@ spec:
               value: /etc/cluster-tree/cert/cert.pem
             - name: APISERVER_KEY_LOCATION
               value: /etc/cluster-tree/cert/key.pem
-            - name: KNODE_POD_IP
+            - name: LEAF_NODE_IP
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: PREFERRED-ADDRESS-TYPE
+              value: InternalDNS
           volumeMounts:
             - name: credentials
               mountPath: "/etc/cluster-tree/cert"

hack/cluster.sh

@@ -12,6 +12,10 @@ KIND_IMAGE="ghcr.io/kosmos-io/kindest/node:v1.25.3_1"
 REUSE=${REUSE:-false}
 VERSION=${VERSION:-latest}
 
+# default cert and key for node server https
+CERT=$(cat ${ROOT}/pkg/cert/crt.pem | base64 -w 0)
+KEY=$(cat ${ROOT}/pkg/cert/key.pem | base64 -w 0)
+
 CN_ZONE=${CN_ZONE:-false}
 
 if [ $REUSE == true ]; then
@@ -163,7 +167,7 @@ function deploy_cluster() {
 
    echo "cluster $clustername deploy clusterlink success"
 
-   sed -e "s|__VERSION__|$VERSION|g" -e "w ${ROOT}/environments/clustertree-cluster-manager.yml" "$ROOT"/deploy/clustertree-cluster-manager.yml
+   sed -e "s|__VERSION__|$VERSION|g" -e "s|__CERT__|$CERT|g" -e "s|__KEY__|$KEY|g" -e "w ${ROOT}/environments/clustertree-cluster-manager.yml" "$ROOT"/deploy/clustertree-cluster-manager.yml
    kubectl --context="kind-${clustername}" apply -f "${ROOT}/environments/clustertree-cluster-manager.yml"
 
    echo "cluster $clustername deploy clustertree success"

pkg/cert/cert.go

@@ -0,0 +1,28 @@
+package cert
+
+import (
+	_ "embed"
+	"encoding/base64"
+)
+
+//go:embed crt.pem
+var crt []byte
+
+//go:embed key.pem
+var key []byte
+
+func GetCrtEncode() string {
+	return base64.StdEncoding.EncodeToString(crt)
+}
+
+func GetKeyEncode() string {
+	return base64.StdEncoding.EncodeToString(key)
+}
+
+func GetCrt() []byte {
+	return crt
+}
+
+func GetKey() []byte {
+	return key
+}

pkg/cert/crt.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDEzCCAfsCFH4WHXLqM/y7lcp+lOzUGymu0kdFMA0GCSqGSIb3DQEBCwUAMEUx
+CzAJBgNVBAYTAlpIMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
+cm5ldCBXaWRnaXRzIFB0eSBMdGQwIBcNMjMxMTIyMDY0MzMwWhgPMzAyMzAzMjUw
+NjQzMzBaMEUxCzAJBgNVBAYTAlpIMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYD
+VQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQCI+Kfjgljh9HaATEzxBjA8YwwsPvkUiR/5lEpKkpF9JpgB
+uYz5F/btwHYncFCmvY60aJLwv4xn06tZeijaNgZf9o5HIOFhosjosi5pCQnUszcQ
+8DJ+5iwXHPrmCOv6ncZCpt2XsbBQ7k/gW4Buzvb+FN85p+n6GsRz3R+JJ62MI4JO
+4QAhtTyyCunCp2mp5kAa0l9iemWkXUV4qW07RcLUgmmsjyEwgsz3hsYjZ/wAaGTp
+GdEBAbhk9/lJSYFy/0TRG/evi/6Ba2jiYTrELa7Y0elrTsL+ulxs7jUH43hQ29VD
+oF3ufcwDJdrOcvQ53c9LRLUh6UIrLFVPZJ1SOeJTAgMBAAEwDQYJKoZIhvcNAQEL
+BQADggEBABWP5dBbHhLw+ppBIWolwkNzEIlBplUooMFotDhNTmsXk5MzSUmu3sJT
+ejR/sLP5HKS644FblpF/8nSdvrPQ8oyfEc91itQT9CS4v8KAF9my7+/6y5iJDxYW
+Cp8lsSvnK1pr766NKF2og+8Z1QrRunCmuc8Vf8UhLpdXAFCygR/oAwc6Y7qrH1Uz
+sijQE3ybRCvreGlLdNTouq2/nlGUvUtbABAd/G2U40xMvP438gOIBlfE4i3if6Ys
+7og6ZbagAcVc+MH3owL6NkYM2dUU2h8w83CPXpML9hHGWpRNtL808q0jLWOxlFVR
+xrUPjT8yrA4b2OeqkntiV9ybTpNVBVk=
+-----END CERTIFICATE-----

pkg/cert/key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCI+Kfjgljh9HaA
+TEzxBjA8YwwsPvkUiR/5lEpKkpF9JpgBuYz5F/btwHYncFCmvY60aJLwv4xn06tZ
+eijaNgZf9o5HIOFhosjosi5pCQnUszcQ8DJ+5iwXHPrmCOv6ncZCpt2XsbBQ7k/g
+W4Buzvb+FN85p+n6GsRz3R+JJ62MI4JO4QAhtTyyCunCp2mp5kAa0l9iemWkXUV4
+qW07RcLUgmmsjyEwgsz3hsYjZ/wAaGTpGdEBAbhk9/lJSYFy/0TRG/evi/6Ba2ji
+YTrELa7Y0elrTsL+ulxs7jUH43hQ29VDoF3ufcwDJdrOcvQ53c9LRLUh6UIrLFVP
+ZJ1SOeJTAgMBAAECggEAGe9DEr/mhnocSfSoiOaMEZMLhgEydmH0bPRYEMCpzZGW
+LJVujOetuJy9goAwtTGlKKG4WN9b/XjFs/5+Z7rdACSWEf+2zR7efbjnMrokY2K/
+pXRli0OXy5SQKSg9Tkm7dXlU8dkSMnC9LRUGP3TurXNURP13PwT8d5fB1d1ubd8w
+bbdcHVvzKZU5T7rdcLBsZ5/70eSqyJstcNDcNr28yI3xY0a2z2NX1XTnRbWXMayF
+PNNOJuoX/mmwxhCkLxyoVsSIz1nlyV3MKXKlDMK2I+LW0LqkvD97Xh3U2P895LTC
+BqRaX3FfFaMjJ5feS/A3CUZdeiPLjT8MWlphTCAR+QKBgQC5YEiwJxV4W+2LfCjD
+32HNIHXWEbvxaS9Ebv93pKolxOqaAptEL81+LRKEhdDRMYQHqHZzHxM75RM0YIYq
+njFLflFWSYskTn+x5z11Re1BS54XgNh05qp7OPpQdBTWkg8fw/eKK3g90fdXDrkn
+7XAK9A/9sJAlAMnVlrqGRDKu5wKBgQC9J3TFJMUt6SNaayFauh6DcJsOq+YSFnVy
+B4hktfI1TkCmW5OPTq/rMUKAe9ejd/ujDtNfpvcnJF0vCTStIeciKZM0s0pAb8nU
+QAxtrsdfPhYJQUO10ycYuiCZZ421/A7QSf4XaZMRUUHO477gFj5MI5n+ysuDwjXn
+m1Arl4/ftQKBgQCe4YErSTRDpjageFfQGWMPlqSoRybYMBjNBH18o+sY1/9i5J0D
+Ah2T6TmXz8E7qr7IeYCcBqRLj3i4SYp0eIUzeR5pYDsbcRRM/C5WlwpUDmV/K3Va
+LGEtn5Ya4oMBrMm9pg5BpCQ4h/7/5KSZLg37tVcHTg8dR+G1aKyRa14tPQKBgGxS
+mymHLDBlkexm63wEmBLXusSFJsV2/R0nOTHLjICAZr+eM/veqRn8ZMQlp9EilgXE
+KMJfYKyWw5J7KCJ6Bt5mhrmobz5FhoS5hSSO8fgWGxKDwJ3w5TPg62hOiDYOugEI
+Tq3jtOg264PqotW7h0OdI8RpKHE1GB+hryC3tBn9AoGAcR1OTKP1S66EcsR/6+o6
+kS9VEAH/4t181f1Km+DGJ5i9GgAQ7OHlqeFZ37JV+MhcjIPVa9lLVGjFTbU9yKun
+hYjbFaAevlvPz95iRWydgYiiEXr877EPS8YO6WzhFXJnSEBNEIkSNKr+ba/860/Z
+MdXk6ivt94ELiDoQMENIq9s=
+-----END PRIVATE KEY-----

pkg/clustertree/cluster-manager/controllers/node_resources_controller.go

@@ -143,13 +143,17 @@ func (c *NodeResourcesController) Reconcile(ctx context.Context, request reconci
 			if node != nil {
 				clone.Labels = mergeMap(node.GetLabels(), clone.GetLabels())
 				clone.Annotations = mergeMap(node.GetAnnotations(), clone.GetAnnotations())
-				spec := corev1.NodeSpec{
-					Taints: rootNode.Spec.Taints,
-				}
-
-				clone.Spec = spec
+				// TODO @duanmengkk
+				// spec := corev1.NodeSpec{
+				// 	Taints: rootNode.Spec.Taints,
+				// }
+				clone.Spec.Taints = rootNode.Spec.Taints
 				clone.Status = node.Status
-				clone.Status.Addresses = leafUtils.GetAddress()
+				clone.Status.Addresses, err = leafUtils.GetAddress(ctx, c.RootClientset, node.Status.Addresses)
+				if err != nil {
+					klog.Errorf("GetAddress node %s, err: %v, ", rootNode.Name, err)
+					return reconcile.Result{}, err
+				}
 			}
 		}
 

pkg/clustertree/cluster-manager/node-server/server.go

@@ -19,6 +19,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/kosmos.io/kosmos/cmd/clustertree/cluster-manager/app/options"
+	"github.com/kosmos.io/kosmos/pkg/cert"
 	"github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/node-server/api"
 	leafUtils "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/utils"
 )
@@ -125,24 +126,31 @@ func (s *NodeServer) AttachRoutes(m *http.ServeMux) {
 	m.Handle("/", r)
 }
 
-func (s *NodeServer) initTLSConfig() (*tls.Config, error) {
+func loadKeyPair() (tls.Certificate, error) {
 	CertPath := os.Getenv("APISERVER_CERT_LOCATION")
 	KeyPath := os.Getenv("APISERVER_KEY_LOCATION")
-	CACertPath := os.Getenv("APISERVER_CA_CERT_LOCATION")
 
+	if CertPath == "" || KeyPath == "" {
+		return tls.X509KeyPair(cert.GetCrt(), cert.GetKey())
+	}
+	return tls.LoadX509KeyPair(CertPath, KeyPath)
+}
+
+func (s *NodeServer) initTLSConfig() (*tls.Config, error) {
 	tlsCfg := &tls.Config{
 		MinVersion:               tls.VersionTLS12,
 		PreferServerCipherSuites: true,
 		CipherSuites:             DefaultServerCiphers(),
 		ClientAuth:               tls.RequestClientCert,
 	}
 
-	cert, err := tls.LoadX509KeyPair(CertPath, KeyPath)
+	cert, err := loadKeyPair()
 	if err != nil {
 		return nil, err
 	}
 	tlsCfg.Certificates = append(tlsCfg.Certificates, cert)
 
+	CACertPath := os.Getenv("APISERVER_CA_CERT_LOCATION")
 	if CACertPath != "" {
 		pem, err := os.ReadFile(CACertPath)
 		if err != nil {

pkg/clustertree/cluster-manager/utils/leaf_model_handler.go

@@ -67,7 +67,7 @@ func (h AggregationModelHandler) CreateNodeInRoot(ctx context.Context, cluster *
 			},
 		}
 
-		node.Status.Addresses = GetAddress()
+		// node.Status.Addresses = GetAddress()
 
 		node, err = h.RootClientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{})
 		if err != nil {
@@ -99,7 +99,24 @@ func (h AggregationModelHandler) UpdateNodeStatus(ctx context.Context, n []*core
 		clone := node.DeepCopy()
 		clone.Status.Conditions = utils.NodeConditions()
 
+		nodeListInLeaf := &corev1.NodeList{}
+		err = h.LeafClient.List(ctx, nodeListInLeaf)
+		if err != nil {
+			return fmt.Errorf("cannot get node in leaf cluster while update node status err: %v", err)
+		}
+
+		if len(nodeListInLeaf.Items) == 0 {
+			return fmt.Errorf("cannot get node in leaf cluster while update node status, leaf node item is 0")
+		}
+
+		clone.Status.Addresses, err = GetAddress(ctx, h.RootClientset, nodeListInLeaf.Items[0].Status.Addresses)
+
+		if err != nil {
+			return err
+		}
+
 		patch, err := utils.CreateMergePatch(node, clone)
+
 		if err != nil {
 			return fmt.Errorf("cannot get node while update node status %s, err: %v", node.Name, err)
 		}
@@ -177,7 +194,7 @@ func (h DispersionModelHandler) CreateNodeInRoot(ctx context.Context, cluster *k
 					},
 				}
 
-				node.Status.Addresses = GetAddress()
+				// node.Status.Addresses = GetAddress()
 
 				node, err = h.RootClientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{})
 				if err != nil {
@@ -214,7 +231,10 @@ func (h DispersionModelHandler) UpdateNodeStatus(ctx context.Context, n []*corev
 
 			rootCopy := nodeRoot.DeepCopy()
 			nodeRoot.Status = nodeInLeaf.Status
-			nodeRoot.Status.Addresses = GetAddress()
+			nodeRoot.Status.Addresses, err = GetAddress(ctx, h.RootClientset, nodeInLeaf.Status.Addresses)
+			if err != nil {
+				return err
+			}
 			nodeRoot.Status.Allocatable = rootCopy.Status.Allocatable
 			nodeRoot.Status.Capacity = rootCopy.Status.Capacity
 

pkg/clustertree/cluster-manager/utils/rootcluster.go

@@ -1,11 +1,17 @@
 package utils
 
 import (
+	"context"
+	"fmt"
 	"os"
+	"sort"
 
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 
 	kosmosv1alpha1 "github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1"
+	"github.com/kosmos.io/kosmos/pkg/utils"
 )
 
 const (
@@ -22,9 +28,76 @@ func IsRootCluster(cluster *kosmosv1alpha1.Cluster) bool {
 	return false
 }
 
-func GetAddress() []corev1.NodeAddress {
-	address := []corev1.NodeAddress{
-		{Type: corev1.NodeInternalIP, Address: os.Getenv("KNODE_POD_IP")},
+func GetAddress(ctx context.Context, rootClient kubernetes.Interface, originAddress []corev1.NodeAddress) ([]corev1.NodeAddress, error) {
+	preferredAddressType := corev1.NodeAddressType(os.Getenv("PREFERRED-ADDRESS-TYPE"))
+
+	if len(preferredAddressType) == 0 {
+		preferredAddressType = corev1.NodeInternalDNS
+	}
+
+	prefixAddress := []corev1.NodeAddress{
+		{Type: preferredAddressType, Address: os.Getenv("KNODE_POD_IP")},
+	}
+
+	address, err := SortAddress(ctx, rootClient, originAddress)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return append(prefixAddress, address...), nil
+}
+
+func SortAddress(ctx context.Context, rootClient kubernetes.Interface, originAddress []corev1.NodeAddress) ([]corev1.NodeAddress, error) {
+	rootnodes, err := rootClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("create node failed, cannot get node from root cluster, err: %v", err)
 	}
-	return address
+
+	if len(rootnodes.Items) == 0 {
+		return nil, fmt.Errorf("create node failed, cannot get node from root cluster, len of leafnodes is 0")
+	}
+
+	isIPv4First := true
+	for _, addr := range rootnodes.Items[0].Status.Addresses {
+		if addr.Type == corev1.NodeInternalIP {
+			if utils.IsIPv6(addr.Address) {
+				isIPv4First = false
+			}
+			break
+		}
+	}
+
+	address := []corev1.NodeAddress{}
+	otherAddress := []corev1.NodeAddress{}
+
+	for _, addr := range originAddress {
+		if addr.Type == corev1.NodeInternalIP {
+			address = append(address, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: addr.Address})
+		} else {
+			otherAddress = append(otherAddress, addr)
+		}
+	}
+
+	sort.Slice(address, func(i, j int) bool {
+		if isIPv4First {
+			if !utils.IsIPv6(address[i].Address) && utils.IsIPv6(address[j].Address) {
+				return true
+			}
+			if utils.IsIPv6(address[i].Address) && !utils.IsIPv6(address[j].Address) {
+				return false
+			}
+			return true
+		} else {
+			if !utils.IsIPv6(address[i].Address) && utils.IsIPv6(address[j].Address) {
+				return false
+			}
+			if utils.IsIPv6(address[i].Address) && !utils.IsIPv6(address[j].Address) {
+				return true
+			}
+			return true
+		}
+	})
+
+	return append(address, otherAddress...), nil
 }

pkg/kosmosctl/install/install.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/kubectl/pkg/util/templates"
 
 	"github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1"
+	"github.com/kosmos.io/kosmos/pkg/cert"
 	"github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned"
 	"github.com/kosmos.io/kosmos/pkg/kosmosctl/join"
 	"github.com/kosmos.io/kosmos/pkg/kosmosctl/manifest"
@@ -64,6 +65,9 @@ type CommandInstallOptions struct {
 	KosmosClient        versioned.Interface
 	K8sClient           kubernetes.Interface
 	K8sExtensionsClient extensionsclient.Interface
+
+	CertEncode string
+	KeyEncode  string
 }
 
 // NewCmdInstall Install the Kosmos control plane in a Kubernetes cluster.
@@ -97,6 +101,9 @@ func NewCmdInstall(f ctlutil.Factory) *cobra.Command {
 	flags.StringVar(&o.UseProxy, "use-proxy", "false", "Set whether to enable proxy.")
 	flags.IntVarP(&o.WaitTime, "wait-time", "", utils.DefaultWaitTime, "Wait the specified time for the Kosmos install ready.")
 
+	flags.StringVar(&o.CertEncode, "cert-encode", cert.GetCrtEncode(), "cert base64 string for node server.")
+	flags.StringVar(&o.KeyEncode, "key-encode", cert.GetKeyEncode(), "key base64 string for node server.")
+
 	return cmd
 }
 
@@ -439,6 +446,23 @@ func (o *CommandInstallOptions) runClustertree() error {
 	}
 	klog.Info("ConfigMap host-kubeconfig has been created.")
 
+	klog.Info("Start creating kosmos-clustertree secret")
+	clustertreeSecret, err := util.GenerateSecret(manifest.ClusterTreeClusterManagerSecret, manifest.SecretReplace{
+		Namespace: o.Namespace,
+		Cert:      o.CertEncode,
+		Key:       o.KeyEncode,
+	})
+	if err != nil {
+		return err
+	}
+	_, err = o.K8sClient.CoreV1().Secrets(o.Namespace).Create(context.Background(), clustertreeSecret, metav1.CreateOptions{})
+	if err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("kosmosctl install clustertree run error, secret options failed: %v", err)
+		}
+	}
+	klog.Info("Secret has been created. ")
+
 	klog.Info("Start creating kosmos-clustertree Deployment...")
 	clustertreeDeploy, err := util.GenerateDeployment(manifest.ClusterTreeClusterManagerDeployment, manifest.DeploymentReplace{
 		Namespace:       o.Namespace,