Merge pull request kosmos-io#259 from OrangeBao/main

feat: kubectl support install and uninstall node server

deploy/clustertree-cluster-manager.yml

@@ -35,8 +35,8 @@ metadata:
   namespace: kosmos-system
 type: Opaque
 data:
-  cert.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQzakNDQXNhZ0F3SUJBZ0lJVWE0NWVxZmI0c0V3RFFZSktvWklodmNOQVFFTEJRQXdmekVMTUFrR0ExVUUKQmhNQ1ZWTXhEekFOQmdOVkJBZ1RCazl5WldkdmJqRVJNQThHQTFVRUJ4TUlVRzl5ZEd4aGJtUXhHREFXQmdOVgpCQW9URDNacmRXSmxiR1YwTFcxdlkyc3RNREVZTUJZR0ExVUVDeE1QZG10MVltVnNaWFF0Ylc5amF5MHdNUmd3CkZnWURWUVFERXc5MmEzVmlaV3hsZEMxdGIyTnJMVEF3SGhjTk1UZ3hNVEkyTVRJd016SXpXaGNOTVRrd01qSTEKTVRnd09ESXpXakIvTVFzd0NRWURWUVFHRXdKVlV6RVBNQTBHQTFVRUNCTUdUM0psWjI5dU1SRXdEd1lEVlFRSApFd2hRYjNKMGJHRnVaREVZTUJZR0ExVUVDaE1QZG10MVltVnNaWFF0Ylc5amF5MHdNUmd3RmdZRFZRUUxFdzkyCmEzVmlaV3hsZEMxdGIyTnJMVEF4R0RBV0JnTlZCQU1URDNacmRXSmxiR1YwTFcxdlkyc3RNRENDQVNJd0RRWUoKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTHJ5SHZLM1VCQkJxR1YyRnB3eW1mMHAvWUtHUUE5cgpOdTBONmYyK1JrVVhMdVFYRytXZEZRbDNaUXliUExmQ0UyaHdGY2wzSUYrM2hDelkzLzJVSXlHQmxvQklmdDdLCllGTE0zWVdKRHk1RWxLRGcxYk5EU0x6RjZ0a3BOTERuVmxna1BQSVR6cEVISUF1K0JUNURaR1doWUFXTy9EaXIKWGR4b0pCT2hQWlpDY0JDVitrd1FRUGJzWHpaeStxN1FoeDI3MENSTUlYc285QzVMSmhHWUw5ZndzeG11a0FPUgo1NlNtZnNBYW1sN1VPbHpISVRSRHdENUFRMUJrVFNFRnkwOGRrNkpBWUw4TERMaGdhTG9Xb1YwR2UyZ09JZXBSCmpwbDg3ZEdiU1ZHeUJIbVRYdjRvNnV0cVQ2UzZuVTc2TG45TlNpN1loTXFqOHVXdjBwVERsWWNDQXdFQUFhTmUKTUZ3d0RnWURWUjBQQVFIL0JBUURBZ1dnTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjRApBakFNQmdOVkhSTUJBZjhFQWpBQU1CMEdBMVVkRGdRV0JCUVZId1Uxc3k3UW53MVd2VnZGTGNacmhvVDQwREFOCkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQXNOR05LejFKd2Z3ZzdyWWFPN1ZGL3phbjAxWFhaRlAxYm5GWW5YSnUKMTVSemhPQk1zcDNLdldDVmh3VWZ4TmU4R2hVRFN4MnRtUzVFQS84b2FFbmdMRmwzanRSM3BuVU5Pd0RWbHpseQpRT0NOM3JsT2k0K3AyNkx2TWlBRnA1aHhYQXYzTE9SczZEenI2aDMvUVR0bFY1akRTaFVPWFpkRmRPUEpkWjJuCmc0Ymlyckc3TU82dnd2UjhDaU5jUTI2YitiOHA5QkdYYkU4YnNKb0htY3NxeWE4ZmJWczJuNkNkRUplSSs0aEQKTjZ4bG81U3Zoakg1dEZJSTdlQ1ZlZHlaR2wwQkt2a29jT2lnTGdxOFgrSnpGeGoxd3RkbXRYdjdzamRLY0I5cgo2VFdHSlJyWlZ4b3hVT3paaHB4VWozai9wTGFSY0RtdHRTSkN1RHUzTkF0a2dRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
-  key.pem: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdXZJZThyZFFFRUdvWlhZV25ES1ovU245Z29aQUQyczI3UTNwL2I1R1JSY3U1QmNiCjVaMFZDWGRsREpzOHQ4SVRhSEFWeVhjZ1g3ZUVMTmpmL1pRaklZR1dnRWgrM3NwZ1VzemRoWWtQTGtTVW9PRFYKczBOSXZNWHEyU2swc09kV1dDUTg4aFBPa1FjZ0M3NEZQa05rWmFGZ0JZNzhPS3RkM0dna0U2RTlsa0p3RUpYNgpUQkJBOXV4Zk5uTDZydENISGJ2UUpFd2hleWowTGtzbUVaZ3YxL0N6R2E2UUE1SG5wS1ord0JxYVh0UTZYTWNoCk5FUEFQa0JEVUdSTklRWExUeDJUb2tCZ3Z3c011R0JvdWhhaFhRWjdhQTRoNmxHT21YenQwWnRKVWJJRWVaTmUKL2lqcTYycFBwTHFkVHZvdWYwMUtMdGlFeXFQeTVhL1NsTU9WaHdJREFRQUJBb0lCQUVOODR0VkdmaDNRUmlXUwpzdWppajVySVROK1E3WkZqYUNtOTZ5b1NSYlh0ZjUwU0JwMG16eEJpek5UM09iMHd6K2JWQjloNksvTENBbkphClBNcURid2RLaS9WMXRtOWhhZEthYUtJcmI1S0phWXFHZ0Q4OTNBVmlBYjB4MWZiREhQV201MldRNXZLT092QmkKUWV4UFVmQXFpTXFZNnM3ZWRuejZENFFTb25RYW14Q1VQQlBZdnVkbWF5SHRQbGM4UWI2ZVkwVitwY2RGblcwOApTRFpYWU94ZXkzL0lBalp5ZGNBN1hndk5TYys2WE93bWhLc0dBVzcxdUZUVGFnSnZ6WDNlUENZMTRya0dKbURHCm0vMTBob1c2Tk1LR2VWL1J5WDNkWDBqSm1EazFWZnhBUVczeHBPaXBaZmdmdmdhdkNPcUhuS0E2SThkSzN6aGcKdkU5QmxlRUNnWUVBODdYL3p0UVpESTRxT1RBOUNXL25NWGZ3QXk5UU8xSzZiR2hCSFV1N0pzNHBxZ3h1SDhGawpoUWdRSzdWOGlhc255L2RDeWo2Q3UzUUpOb2Z4dWRBdkxMUUtrcXV5UU9hK3pxRkNVcFZpZDdKVlJNY1JMSmx0CjNIbHlDTnZWbGhmakRUMGNJMlJkVTQ1cThNblpveTFmM0RQWkIxNmNIYjNITDl6MWdRWlRpWEVDZ1lFQXhGOWEKNjhTYnhtV0ZCSzdQYW9iSTh3VmZEb1Rpckhtb0F2bnlwWUswb1FrQVg4Vm1FbXRFRXMyK04xbW9LalNUUHIrdAp1czRKS2d1QTh6MnR1TGs1aitlRit6RGwvMlUrN2RqVEY4RkNOcHJ3ejNzWHI0MjdHQ0lHTDVZdnBJQlorVEw4CkJqaTJ1eW9vOGs5U0FXTWI0T2JPemZHbTR0ZUN2Y2lTOTlndzBuY0NnWUF0NUdiQVZ0WkVzL3lsZWp6MEt2dFoKS0dHczU5cnU0TncwRDhtN0w0aVZmUnNCWjRmUk9RU3B2R1AzSnh6RmU5SnBxUzBOa29uaHJLOFRjclFGTG52RApxaitYY1BlSEd5eHhFcEsvcEZ1L2VIaHdGQ0JheXFXU2I5Z1diUGNpWldzZkVoUGJZa25rc3h2V0xkeHF5dCtUClFyd3FsQmxIekhYV3dJQUdoTjkwTVFLQmdRQzVDWWtwQkZnc3VGaUJNeCtySjFxTzlJNi9wYVBhRmNDbEhWVHgKZEpvejY4RjRmUTlUWjlQN1MvZGpQSTVqUnF0QXcyazJ6eEovbGR0cVdNSXJnQTJuZGVnZjY5R3R1SDkxcTR3dApwQ042Uk1HSklGb1BTQ1AxOTRtUXFabzNEZUs2R0xxMk9oYWxnbktXOFBzNjUyTExwM0ZUU2RPUmlMVmZrM0k1CkxIUEV2UUtCZ0RDeGEvM3ZuZUc4dmdzOEFyRWpOODlCL1l4TzFxSVU1bXhKZTZaYWZiODFOZGhZVWpmUkFWcm8KQUxUb2ZpQXBNc25EYkpESE1pd3Z3Y0RVSGJQTHBydUs4MFIvL3ptWDdYZW4rRis1b2JmU1E4ajBHU21tZVdGUQpTVkc2QXBOdGt0TFBJMG5LMm5FSUgvUXg0b3VHQzlOMHBBRFJDbFFRUFN4RVBtRHZmNHhmCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCgo=
+  cert.pem: __CERT__
+  key.pem: __KEY__
 
 ---
 apiVersion: apps/v1
@@ -66,7 +66,7 @@ spec:
               value: /etc/cluster-tree/cert/cert.pem
             - name: APISERVER_KEY_LOCATION
               value: /etc/cluster-tree/cert/key.pem
-            - name: KNODE_POD_IP
+            - name: LEAF_NODE_IP
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP

hack/cluster.sh

@@ -12,6 +12,10 @@ KIND_IMAGE="ghcr.io/kosmos-io/kindest/node:v1.25.3_1"
 REUSE=${REUSE:-false}
 VERSION=${VERSION:-latest}
 
+# default cert and key for node server https
+CERT=$(cat ${ROOT}/pkg/cert/crt.pem | base64 -w 0)
+KEY=$(cat ${ROOT}/pkg/cert/key.pem | base64 -w 0)
+
 CN_ZONE=${CN_ZONE:-false}
 
 if [ $REUSE == true ]; then
@@ -163,7 +167,7 @@ function deploy_cluster() {
 
    echo "cluster $clustername deploy clusterlink success"
 
-   sed -e "s|__VERSION__|$VERSION|g" -e "w ${ROOT}/environments/clustertree-cluster-manager.yml" "$ROOT"/deploy/clustertree-cluster-manager.yml
+   sed -e "s|__VERSION__|$VERSION|g" -e "s|__CERT__|$CERT|g" -e "s|__KEY__|$KEY|g" -e "w ${ROOT}/environments/clustertree-cluster-manager.yml" "$ROOT"/deploy/clustertree-cluster-manager.yml
    kubectl --context="kind-${clustername}" apply -f "${ROOT}/environments/clustertree-cluster-manager.yml"
 
    echo "cluster $clustername deploy clustertree success"

pkg/cert/cert.go

@@ -0,0 +1,28 @@
+package cert
+
+import (
+	_ "embed"
+	"encoding/base64"
+)
+
+//go:embed crt.pem
+var crt []byte
+
+//go:embed key.pem
+var key []byte
+
+func GetCrtEncode() string {
+	return base64.StdEncoding.EncodeToString(crt)
+}
+
+func GetKeyEncode() string {
+	return base64.StdEncoding.EncodeToString(key)
+}
+
+func GetCrt() []byte {
+	return crt
+}
+
+func GetKey() []byte {
+	return key
+}

pkg/cert/crt.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDEzCCAfsCFH4WHXLqM/y7lcp+lOzUGymu0kdFMA0GCSqGSIb3DQEBCwUAMEUx
+CzAJBgNVBAYTAlpIMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
+cm5ldCBXaWRnaXRzIFB0eSBMdGQwIBcNMjMxMTIyMDY0MzMwWhgPMzAyMzAzMjUw
+NjQzMzBaMEUxCzAJBgNVBAYTAlpIMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYD
+VQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQCI+Kfjgljh9HaATEzxBjA8YwwsPvkUiR/5lEpKkpF9JpgB
+uYz5F/btwHYncFCmvY60aJLwv4xn06tZeijaNgZf9o5HIOFhosjosi5pCQnUszcQ
+8DJ+5iwXHPrmCOv6ncZCpt2XsbBQ7k/gW4Buzvb+FN85p+n6GsRz3R+JJ62MI4JO
+4QAhtTyyCunCp2mp5kAa0l9iemWkXUV4qW07RcLUgmmsjyEwgsz3hsYjZ/wAaGTp
+GdEBAbhk9/lJSYFy/0TRG/evi/6Ba2jiYTrELa7Y0elrTsL+ulxs7jUH43hQ29VD
+oF3ufcwDJdrOcvQ53c9LRLUh6UIrLFVPZJ1SOeJTAgMBAAEwDQYJKoZIhvcNAQEL
+BQADggEBABWP5dBbHhLw+ppBIWolwkNzEIlBplUooMFotDhNTmsXk5MzSUmu3sJT
+ejR/sLP5HKS644FblpF/8nSdvrPQ8oyfEc91itQT9CS4v8KAF9my7+/6y5iJDxYW
+Cp8lsSvnK1pr766NKF2og+8Z1QrRunCmuc8Vf8UhLpdXAFCygR/oAwc6Y7qrH1Uz
+sijQE3ybRCvreGlLdNTouq2/nlGUvUtbABAd/G2U40xMvP438gOIBlfE4i3if6Ys
+7og6ZbagAcVc+MH3owL6NkYM2dUU2h8w83CPXpML9hHGWpRNtL808q0jLWOxlFVR
+xrUPjT8yrA4b2OeqkntiV9ybTpNVBVk=
+-----END CERTIFICATE-----

pkg/cert/key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCI+Kfjgljh9HaA
+TEzxBjA8YwwsPvkUiR/5lEpKkpF9JpgBuYz5F/btwHYncFCmvY60aJLwv4xn06tZ
+eijaNgZf9o5HIOFhosjosi5pCQnUszcQ8DJ+5iwXHPrmCOv6ncZCpt2XsbBQ7k/g
+W4Buzvb+FN85p+n6GsRz3R+JJ62MI4JO4QAhtTyyCunCp2mp5kAa0l9iemWkXUV4
+qW07RcLUgmmsjyEwgsz3hsYjZ/wAaGTpGdEBAbhk9/lJSYFy/0TRG/evi/6Ba2ji
+YTrELa7Y0elrTsL+ulxs7jUH43hQ29VDoF3ufcwDJdrOcvQ53c9LRLUh6UIrLFVP
+ZJ1SOeJTAgMBAAECggEAGe9DEr/mhnocSfSoiOaMEZMLhgEydmH0bPRYEMCpzZGW
+LJVujOetuJy9goAwtTGlKKG4WN9b/XjFs/5+Z7rdACSWEf+2zR7efbjnMrokY2K/
+pXRli0OXy5SQKSg9Tkm7dXlU8dkSMnC9LRUGP3TurXNURP13PwT8d5fB1d1ubd8w
+bbdcHVvzKZU5T7rdcLBsZ5/70eSqyJstcNDcNr28yI3xY0a2z2NX1XTnRbWXMayF
+PNNOJuoX/mmwxhCkLxyoVsSIz1nlyV3MKXKlDMK2I+LW0LqkvD97Xh3U2P895LTC
+BqRaX3FfFaMjJ5feS/A3CUZdeiPLjT8MWlphTCAR+QKBgQC5YEiwJxV4W+2LfCjD
+32HNIHXWEbvxaS9Ebv93pKolxOqaAptEL81+LRKEhdDRMYQHqHZzHxM75RM0YIYq
+njFLflFWSYskTn+x5z11Re1BS54XgNh05qp7OPpQdBTWkg8fw/eKK3g90fdXDrkn
+7XAK9A/9sJAlAMnVlrqGRDKu5wKBgQC9J3TFJMUt6SNaayFauh6DcJsOq+YSFnVy
+B4hktfI1TkCmW5OPTq/rMUKAe9ejd/ujDtNfpvcnJF0vCTStIeciKZM0s0pAb8nU
+QAxtrsdfPhYJQUO10ycYuiCZZ421/A7QSf4XaZMRUUHO477gFj5MI5n+ysuDwjXn
+m1Arl4/ftQKBgQCe4YErSTRDpjageFfQGWMPlqSoRybYMBjNBH18o+sY1/9i5J0D
+Ah2T6TmXz8E7qr7IeYCcBqRLj3i4SYp0eIUzeR5pYDsbcRRM/C5WlwpUDmV/K3Va
+LGEtn5Ya4oMBrMm9pg5BpCQ4h/7/5KSZLg37tVcHTg8dR+G1aKyRa14tPQKBgGxS
+mymHLDBlkexm63wEmBLXusSFJsV2/R0nOTHLjICAZr+eM/veqRn8ZMQlp9EilgXE
+KMJfYKyWw5J7KCJ6Bt5mhrmobz5FhoS5hSSO8fgWGxKDwJ3w5TPg62hOiDYOugEI
+Tq3jtOg264PqotW7h0OdI8RpKHE1GB+hryC3tBn9AoGAcR1OTKP1S66EcsR/6+o6
+kS9VEAH/4t181f1Km+DGJ5i9GgAQ7OHlqeFZ37JV+MhcjIPVa9lLVGjFTbU9yKun
+hYjbFaAevlvPz95iRWydgYiiEXr877EPS8YO6WzhFXJnSEBNEIkSNKr+ba/860/Z
+MdXk6ivt94ELiDoQMENIq9s=
+-----END PRIVATE KEY-----

pkg/clustertree/cluster-manager/node-server/server.go

@@ -19,6 +19,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/kosmos.io/kosmos/cmd/clustertree/cluster-manager/app/options"
+	"github.com/kosmos.io/kosmos/pkg/cert"
 	"github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/node-server/api"
 	leafUtils "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/utils"
 )
@@ -125,24 +126,31 @@ func (s *NodeServer) AttachRoutes(m *http.ServeMux) {
 	m.Handle("/", r)
 }
 
-func (s *NodeServer) initTLSConfig() (*tls.Config, error) {
+func loadKeyPair() (tls.Certificate, error) {
 	CertPath := os.Getenv("APISERVER_CERT_LOCATION")
 	KeyPath := os.Getenv("APISERVER_KEY_LOCATION")
-	CACertPath := os.Getenv("APISERVER_CA_CERT_LOCATION")
 
+	if CertPath == "" || KeyPath == "" {
+		return tls.X509KeyPair(cert.GetCrt(), cert.GetKey())
+	}
+	return tls.LoadX509KeyPair(CertPath, KeyPath)
+}
+
+func (s *NodeServer) initTLSConfig() (*tls.Config, error) {
 	tlsCfg := &tls.Config{
 		MinVersion:               tls.VersionTLS12,
 		PreferServerCipherSuites: true,
 		CipherSuites:             DefaultServerCiphers(),
 		ClientAuth:               tls.RequestClientCert,
 	}
 
-	cert, err := tls.LoadX509KeyPair(CertPath, KeyPath)
+	cert, err := loadKeyPair()
 	if err != nil {
 		return nil, err
 	}
 	tlsCfg.Certificates = append(tlsCfg.Certificates, cert)
 
+	CACertPath := os.Getenv("APISERVER_CA_CERT_LOCATION")
 	if CACertPath != "" {
 		pem, err := os.ReadFile(CACertPath)
 		if err != nil {

pkg/kosmosctl/install/install.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/kubectl/pkg/util/templates"
 
 	"github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1"
+	"github.com/kosmos.io/kosmos/pkg/cert"
 	"github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned"
 	"github.com/kosmos.io/kosmos/pkg/kosmosctl/join"
 	"github.com/kosmos.io/kosmos/pkg/kosmosctl/manifest"
@@ -64,6 +65,9 @@ type CommandInstallOptions struct {
 	KosmosClient        versioned.Interface
 	K8sClient           kubernetes.Interface
 	K8sExtensionsClient extensionsclient.Interface
+
+	CertEncode string
+	KeyEncode  string
 }
 
 // NewCmdInstall Install the Kosmos control plane in a Kubernetes cluster.
@@ -97,6 +101,9 @@ func NewCmdInstall(f ctlutil.Factory) *cobra.Command {
 	flags.StringVar(&o.UseProxy, "use-proxy", "false", "Set whether to enable proxy.")
 	flags.IntVarP(&o.WaitTime, "wait-time", "", utils.DefaultWaitTime, "Wait the specified time for the Kosmos install ready.")
 
+	flags.StringVar(&o.CertEncode, "cert-encode", cert.GetCrtEncode(), "cert base64 string for node server.")
+	flags.StringVar(&o.KeyEncode, "key-encode", cert.GetKeyEncode(), "key base64 string for node server.")
+
 	return cmd
 }
 
@@ -439,6 +446,23 @@ func (o *CommandInstallOptions) runClustertree() error {
 	}
 	klog.Info("ConfigMap host-kubeconfig has been created.")
 
+	klog.Info("Start creating kosmos-clustertree secret")
+	clustertreeSecret, err := util.GenerateSecret(manifest.ClusterTreeClusterManagerSecret, manifest.SecretReplace{
+		Namespace: o.Namespace,
+		Cert:      o.CertEncode,
+		Key:       o.KeyEncode,
+	})
+	if err != nil {
+		return err
+	}
+	_, err = o.K8sClient.CoreV1().Secrets(o.Namespace).Create(context.Background(), clustertreeSecret, metav1.CreateOptions{})
+	if err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("kosmosctl install clustertree run error, secret options failed: %v", err)
+		}
+	}
+	klog.Info("Secret has been created. ")
+
 	klog.Info("Start creating kosmos-clustertree Deployment...")
 	clustertreeDeploy, err := util.GenerateDeployment(manifest.ClusterTreeClusterManagerDeployment, manifest.DeploymentReplace{
 		Namespace:       o.Namespace,

pkg/kosmosctl/manifest/manifest_deployments.go

@@ -120,10 +120,27 @@ spec:
         - name: manager
           image: {{ .ImageRepository }}/clustertree-cluster-manager:v{{ .Version }}
           imagePullPolicy: IfNotPresent
+          env:
+            - name: APISERVER_CERT_LOCATION
+              value: /etc/cluster-tree/cert/cert.pem
+            - name: APISERVER_KEY_LOCATION
+              value: /etc/cluster-tree/cert/key.pem
+            - name: LEAF_NODE_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          volumeMounts:
+            - name: credentials
+              mountPath: "/etc/cluster-tree/cert"
+              readOnly: true
           command:
             - clustertree-cluster-manager
             - --multi-cluster-service=true
             - --v=4
+      volumes:
+        - name: credentials
+          secret:
+            secretName: clustertree-cluster-manager
 `
 
 	CorednsDeployment = `

pkg/kosmosctl/manifest/manifest_secrets.go

@@ -0,0 +1,21 @@
+package manifest
+
+const (
+	ClusterTreeClusterManagerSecret = `---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: clustertree-cluster-manager
+  namespace: {{ .Namespace }}
+type: Opaque
+data:
+  cert.pem: {{ .Cert }}
+  key.pem: {{ .Key }}
+`
+)
+
+type SecretReplace struct {
+	Namespace string
+	Cert      string
+	Key       string
+}

pkg/kosmosctl/uninstall/uninstall.go

@@ -319,6 +319,22 @@ func (o *CommandUninstallOptions) runClustertree() error {
 		}
 	} else {
 		klog.Info("Deployment " + clustertreeDeploy.Name + " is deleted.")
+		clustertreeSecret, err := util.GenerateService(manifest.ClusterTreeClusterManagerSecret, manifest.SecretReplace{
+			Namespace: o.Namespace,
+			Cert:      "",
+			Key:       "",
+		})
+		if err != nil {
+			return err
+		}
+		err = o.K8sClient.CoreV1().Secrets(o.Namespace).Delete(context.Background(), clustertreeSecret.Name, metav1.DeleteOptions{})
+		if err != nil {
+			if !apierrors.IsNotFound(err) {
+				return fmt.Errorf("kosmosctl uninstall clustertree secret run error, secret options failed: %v", err)
+			}
+		} else {
+			klog.Info("Secret " + clustertreeSecret.Name + " is deleted.")
+		}
 	}
 
 	clusters, err := o.KosmosClient.KosmosV1alpha1().Clusters().List(context.TODO(), metav1.ListOptions{})

pkg/kosmosctl/util/builder.go

@@ -179,3 +179,20 @@ func GenerateService(template string, obj interface{}) (*corev1.Service, error)
 	}
 	return o, nil
 }
+
+func GenerateSecret(template string, obj interface{}) (*corev1.Secret, error) {
+	bs, err := parseTemplate(template, obj)
+	if err != nil {
+		return nil, fmt.Errorf("kosmosctl parsing secret template exception, error: %v", err)
+	} else if bs == nil {
+		return nil, fmt.Errorf("kosmosctl get secret template exception, value is empty")
+	}
+
+	o := &corev1.Secret{}
+
+	if err = runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), bs, o); err != nil {
+		return nil, fmt.Errorf("kosmosctl decode secret bytes error: %v", err)
+	}
+
+	return o, nil
+}